Here is my previous post Golang template.ParseFiles "not a directory" error. I have snippet:
root_path, err := osext.Executable()
if err != nil {
return err
}
templates_path := root_path + "/app/views/mailtemplates/" + "feedback"
text_path := templates_path + ".txt"
textTmpl, err := template.ParseFiles(text_path)
if err != nil {
return err
}
and next error:
open /home/cnaize/gocode/bin/advorts/app/views/mailtemplates/feedback.txt: not a directory
If I hardcode to:
templates_path := "/home/cnaize/" + "feedback"
everything is ok.
Where is the problem? I've tried to remove my bin, but it didn't help.
EDITED:
My env variables:
export GOROOT="/usr/local/go"
export GOPATH=$HOME/gocode
export PATH=$PATH:$GOROOT/bin
export PATH="$PATH:$GOPATH/bin"
and my PATH variable:
PATH=/home/cnaize/.rvm/gems/ruby-2.1.0/bin:/home/cnaize/.rvm/gems/ruby-2.1.0#global/bin:/home/cnaize/.rvm/rubies/ruby-2.1.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/cnaize/.rvm/bin:/usr/local/go/bin:/home/cnaize/gocode/bin
As per your fix, the right way to refer to the embedded templates/test.txt resource would be (using filepath.Dir):
root_path, err := osext.Executable()
template_path := filepath.Join(filepath.Dir(root_path), "templates", "test")
text_path := template_path + ".txt"
You would find a similar approach in cmd/go/build.go, for getting the package files:
func (gccgoToolchain) pkgpath(basedir string, p *Package) string {
end := filepath.FromSlash(p.ImportPath + ".a")
afile := filepath.Join(basedir, end)
// add "lib" to the final element
return filepath.Join(filepath.Dir(afile), "lib"+filepath.Base(afile))
}
Related
I need to update a secret with specific value,(the secret contain additional data) my question is how I can update just the value and not all the secret data (I don't want to override the existing data). I mean if the secret have additional values I don’t want to override them just the entry foo
updSec := v1.Secret{
TypeMeta: metav1.TypeMeta{},
ObjectMeta: metav1.ObjectMeta{
Name: "d-values",
Namespace: "terv”,
},
Immutable: nil,
Data: nil,
StringData: nil,
Type: "Opaque",
}
updSec.Data[“foo”] = newVal
if err := r.Client.Update(ctx, &updSec); err != nil {
return ctrl.Result{}, err
}
The issue is that the secret is already exist and here im creating new object and not sure how to do it right ...I need for secret that called d-values just update the newVal for key foo
update
when trying the code in the answer after I run the
patch, err := yaml.Marshal(updSec)
the data looks like following
and the patch are failed with error, any idea if its related ?
if I try with the c.Client.Update it works but not with Patch but the Patch is the right way as if I've ties before is should keep them..
I don't think you can update a single key using the Update method, but you can certainly do that using Patch instead. Here's an example that uses a StrategicMergePatch; it will replace the key val2 in a secret with the value newval:
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"path/filepath"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
)
func main() {
var kubeconfig *string
var namespace *string
var secretname *string
namespace = flag.String("namespace", "", "namespace of secret")
secretname = flag.String("name", "", "name of secret")
if home := homedir.HomeDir(); home != "" {
kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
} else {
kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
}
flag.Parse()
if *namespace == "" {
panic(fmt.Errorf("you must specify a namespace"))
}
if *secretname == "" {
panic(fmt.Errorf("you must specify a secret name"))
}
config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
if err != nil {
panic(err)
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
panic(err)
}
secretClient := clientset.CoreV1().Secrets(*namespace)
ctx := context.TODO()
updSec := v1.Secret{
Data: map[string][]byte{
"val2": []byte("newval"),
},
}
payloadBytes, err := json.Marshal(updSec)
if err != nil {
panic(err)
}
if _, err = secretClient.Patch(ctx, *secretname,
types.StrategicMergePatchType, payloadBytes, metav1.PatchOptions{}); err != nil {
panic(err)
}
// Fetch updated secret
sec, err := secretClient.Get(ctx, *secretname, metav1.GetOptions{})
if err != nil {
panic(err)
}
secJson, err := json.MarshalIndent(sec, "", " ")
if err != nil {
panic(err)
}
fmt.Print(string(secJson))
}
For example, if I create a secret like this:
kubectl create secret generic \
--from-literal val1=key1 \
--from-literal val2=key2 example
And then run the above code like this:
go run main.go -namespace default -name example
The code will output the update secret. Looking at the data section, we see:
"data": {
"val1": "a2V5MQ==",
"val2": "bmV3dmFs"
},
And if we decode val2 we see:
$ kubectl get secret example -o json | jq '.data.val2|#base64d'
"newval"
Using the Operator SDK
If you're working with the Operator SDK, you can use Update if you're first reading the existing value, like this:
// Read the existing secret
secret := &corev1.Secret{}
if err := r.Get(ctx, req.NamespacedName, secret); err != nil {
panic(err)
}
// Check if it needs to be modified
val, ok := secret.Data["val2"]
// If yes, update the secret with a new value and then write
// the entire object back with Update
if !ok || !bytes.Equal(val, []byte("val2")) {
ctxlog.Info("needs update", "secret", secret)
secret.Data["val2"] = []byte("newval")
if err := r.Update(ctx, secret); err != nil {
panic(err)
}
}
You can use the Patch method if you only want to submit a partial update:
if !ok || !bytes.Equal(val, []byte("val2")) {
ctxlog.Info("needs update", "secret", secret)
newVal := corev1.Secret{
Data: map[string][]byte{
"val2": []byte("newval"),
},
}
patch, err := json.Marshal(newVal)
if err != nil {
panic(err)
}
if err := r.Client.Patch(ctx, secret, client.RawPatch(types.StrategicMergePatchType, patch)); err != nil {
panic(err)
}
}
This is pretty much identical to the earlier example. There are examples of using the client.Patch method in the docs, but I'll be honest, I don't find the example very clear.
My program is as follows as a whole.
func main() {
flag.Parse()
if *token == "" {
log.Fatal(Red + "please provide a client token => -token={$token}")
}
tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: *token})
oauthClient := oauth2.NewClient(context.TODO(), tokenSource)
client := putio.NewClient(oauthClient)
//paths := make(chan string)
var wg = new(sync.WaitGroup)
for i := 0; i < 50; i++ {
wg.Add(1)
go worker(paths, wg, client)
}
WalkFilePath()
//if err := filepath.Walk(*rootpath, func(path string, info os.FileInfo, err error) error {
// if err != nil {
// return fmt.Errorf("Failed to walk directory: %T %w", err, err)
// }
// if !info.IsDir() {
// paths <- path
// }
// return nil
//}); err != nil {
// panic(fmt.Errorf("failed Walk: %w", err))
//}
close(paths)
wg.Wait()
}
// walks the file path and sends paths to channel
func WalkFilePath() {
if err := filepath.Walk(*rootpath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return fmt.Errorf("Failed to walk directory: %T %w", err, err)
}
if !info.IsDir() {
paths <- path
}
return nil
}); err != nil {
panic(fmt.Errorf("failed Walk: %w", err))
}
}
func worker(paths <-chan string, wg *sync.WaitGroup, client *putio.Client) {
defer wg.Done()
for path := range paths {
f, err := os.Open(path)
if err != nil {
log.Printf(Red + "Failed to open file %v for reading" + Reset, f.Name())
}
upload, err := client.Files.Upload(context.TODO(), f, path, 0)
if err != nil {
log.Printf(Red + "Failed to upload file %v" + Reset, upload.File.Name)
}
log.Printf(Green+ "File %v has been uploaded succesfully" + Reset, upload.File.Name)
}
}
I did write the code. That's the cleanest I can do and I was told to write a unit test for the program. I'm confused. For example, considering the WalkFilePath function. What should I provide and what kind of result I should expect to test the function. Because it contains channel communication meaning goroutines. Is there any way to write unit tests for this program clearly? Or should I change the code structure which is not good in this case for me. Btw, the program runs properly.
Like most things, Go is very opinionated about how to test. Make sure to read https://go.dev/doc/tutorial/add-a-test
For example, considering the WalkFilePath function. What should I provide and what kind of result I should expect to test the function.
The input to WalkFilePath should be paths and a rootpath. Your WalkFilePath doesn't get paths or rootpath from anywhere, so this code wouldn't compile as is (testing will help catch that stuff of course).
A test for WalkFilePath might be done something like this:
Create a filesystem structure in your project under testdata/, a directory expressly set aside for data used for testing. Create subdirectories and files. For an example that might look like:
testdata/
walktest/
dir1/
file1.txt
dir2/
file2.txt
dir3/
file3.txt
Now you can define the expected data you'll be getting out of your channel.
expected_paths := []string{
"testdata/walktest/dir1/file1.txt",
"testdata/walktest/dir2/file2.txt",
"testdata/walktest/dir3/file3.txt"
}
Now you need to change WalkFilePath to take arguments for rootpath and paths.
func WalkFilePath(rootdir string, paths chan<- string) {
Now you're ready to write your test.
func TestWalkFilePath(t *testing.T(
paths := make(chan string)
go WalkFilePath("testdata/walktest")
results := make([]string,0)
for path := range paths {
results = append(results, path)
}
exp, res := strings.Join(expected_paths, ""), strings.Join(results, "")
if exp != res {
t.Errorf("Expected %s got %s", exp, res)
}
}
Because it contains channel communication meaning goroutines.
It's totally normal and valid to use channels and goroutines in unit tests.
I have created a temp dir using tmpDir, err := ioutil.TempDir(dir, "OAS"). And i used this path to add a swagger extracted from aws to this temp dir. path = tmpDir + "/" + apiName + ".json", but it doesnt work. i also tried path = <path>/OAS/apiName.json it didn't work either. So my question is if I want add a file to this tempDir how do I define it's path?
cmd_3, err := exec.Command("aws", "apigateway", "get-export", "--rest-api-id", api_id, "--stage-name", stageName, "--export-type", "swagger", path).Output()
pwd, err := os.Getwd()
if err != nil {
return
}
dir = pwd
} //gets the path where the program is executed from
apiName := flagApiNameToGet
stageName := flagStageName
path = tmpDir + "/" + apiName + ".json"
// Searching for API ID:
for _, item := range apis.Items {
if item.Name == apiName {
fmt.Printf("API ID found: %+v ", item.Id)
api_id := item.Id
cmd_3, err := exec.Command("aws", "apigateway", "get-export", "--rest-api-id", api_id, "--stage-name", stageName, "--export-type", "swagger", path).Output()
if err != nil {
return err
}
output := string(cmd_3[:])
fmt.Println(output)
found = true
break
}
}
func execute() {
tmpDir, err := ioutil.TempDir(dir, "OAS")
if err != nil {
fmt.Println("Error creating temporary directory to store OAS")
return
}
fmt.Println("Temporary directory created:", tmpDir)
defer os.RemoveAll(tmpDir)
err = getOAS()
if err != nil {
utils.HandleErrorAndExit("Error getting OAS from AWS. ", err)
}
err = initializeProject()
if err != nil {
utils.HandleErrorAndExit("Error initializing project. ", err)
}
fmt.Println("Temporary directory deleted")
}
Since the tmpDir variable is global. Change your code to:
var err error
tmpDir, err = ioutil.TempDir(dir, "OAS")
Spotted the difference? := and =. Other fuction doesnt see scope declared variable tmpDir.
Here is an example of your code playground as you can see the global var dir is empty in other function call. Fixed version
I made a program in Python 2.7 that renames the files in a directory by replacing the numbers in those file names. I tried to do the same thing with a Golang app but it is not working. The console still gives be new names but it is not changing the names of the photos in the file directory.
Here is the python program
import os
def rename_files():
#(1) get files names from a folder
file_list = os.listdir(r"C:\\Users\\g\\Desktop\\Fun\\udacity\\foundationsofpython\\07finalrenamingapp\\prank")
print(file_list)
saved_path = os.getcwd()
print (saved_path)
os.chdir(r"C:\\Users\\g\\Desktop\\Fun\\udacity\\foundationsofpython\\07finalrenamingapp\\prank")
#2 rename all files names in folder
for file_name in file_list:
print ("Old Name - " +file_name)
print ("New Name - " +file_name.strip("0123456789"))
os.rename(file_name,file_name.strip("0123456789"))
os.chdir(saved_path)
rename_files()
This works just fine. However, this Golang script does not
This is the Golang program
package main
import (
"log"
"os"
"fmt"
"regexp"
)
func readCurrentDir() {
dir := "C:\\Users\\g\\Desktop\\prank"
file, err := os.Open(dir)
if err != nil {
log.Fatalf("failed opening directory: %s", err)
}
defer file.Close()
list,_ := file.Readdirnames(0) // 0 to read all files and folders
for _, name := range list {
oldName := name
fmt.Println("Old Name - ", oldName)
re := regexp.MustCompile( "[^A-za-z]" )
newName := re.ReplaceAllString( oldName, " ")
fmt.Println("New Name - ", newName)
os.Rename(dir+oldName, dir+newName)
fmt.Println("File names have been changed")
}
}
func main() {
readCurrentDir()
}
If you want to pull these to projects from the github repo here is a link that will provide you with the code and the photo folder where the images are stored. I would benefit the most by seeing where my code needs to be changed. Any help or advice you could give would be greatly appreciated. Thank you very much.
Python App https://github.com/lashleykeith/GoingwithGolang/tree/master/OldCode2BeChanged
Golang App
https://github.com/lashleykeith/GoingwithGolang/tree/master/our_tutorial/5finalrenamingapp
In Go, always, always, check for errors. Many of your problems will then be obvious. For example, you have os.Rename errors.
Use the filepath package to manipulate filename paths. Compiling a constant regex expression should only be done once. Fix your regexp bug: "[^A-za-z]" should be "[^A-Za-z]".
Here's a working version:
package main
import (
"fmt"
"log"
"os"
"path/filepath"
"regexp"
)
func readCurrentDir() {
dir := "C:\\Users\\g\\Desktop\\prank"
file, err := os.Open(dir)
if err != nil {
log.Fatalf("failed opening directory: %s", err)
}
defer file.Close()
list, err := file.Readdirnames(0) // 0 to read all files and folders
if err != nil {
log.Fatalf("failed reading directory: %s", err)
}
re := regexp.MustCompile("[^A-Za-z]")
for _, name := range list {
oldName := name
fmt.Println("Old Name - ", oldName)
newName := re.ReplaceAllString(oldName, " ")
fmt.Println("New Name - ", newName)
err := os.Rename(filepath.Join(dir, oldName), filepath.Join(dir, newName))
if err != nil {
log.Printf("error renaming file: %s", err)
continue
}
fmt.Println("File names have been changed")
}
}
func main() {
readCurrentDir()
}
Output:
Old Name - prankster.42.txt
New Name - prankster txt
File names have been changed
Diff:
> git diff old.go new.go
diff --git a/old.go b/new.go
index 464b60e..7ae276e 100644
--- a/old.go
+++ b/new.go
## -4,6 +4,7 ## import (
"fmt"
"log"
"os"
+ "path/filepath"
"regexp"
)
## -15,69 +16,31 ## func readCurrentDir() {
}
defer file.Close()
- list, _ := file.Readdirnames(0) // 0 to read all files and folders
+ list, err := file.Readdirnames(0) // 0 to read all files and folders
+ if err != nil {
+ log.Fatalf("failed reading directory: %s", err)
+ }
+ re := regexp.MustCompile("[^A-Za-z]")
for _, name := range list {
oldName := name
fmt.Println("Old Name - ", oldName)
- re := regexp.MustCompile("[^A-za-z]")
newName := re.ReplaceAllString(oldName, " ")
fmt.Println("New Name - ", newName)
- os.Rename(dir+oldName, dir+newName)
+ err := os.Rename(filepath.Join(dir, oldName), filepath.Join(dir, newName))
+ if err != nil {
+ log.Printf("error renaming file: %s", err)
+ continue
+ }
fmt.Println("File names have been changed")
-
}
}
func main() {
readCurrentDir()
-
}
>
I'm using the script from this post to download files from s3.
Everything works fine except the downloaded files are all 0B.
My script are basically the same as the script above.
I use ~/.aws/credentials to set my keys and set region in the script.
Here is my script:
package main
import (
"fmt"
"os"
"path/filepath"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
var (
Bucket = "logs" // Download from this bucket
Prefix = "local-dir/my_log/20150611/20150611" // Using this key prefix
LocalDirectory = "s3logs" // Into this directory
)
func main() {
manager := s3manager.NewDownloader(nil)
d := downloader{bucket: Bucket, dir: LocalDirectory, Downloader: manager}
client := s3.New(&aws.Config{Region: "ap-northeast-1"})
params := &s3.ListObjectsInput{Bucket: &Bucket, Prefix: &Prefix}
client.ListObjectsPages(params, d.eachPage)
}
type downloader struct {
*s3manager.Downloader
bucket, dir string
}
func (d *downloader) eachPage(page *s3.ListObjectsOutput, more bool) bool {
for _, obj := range page.Contents {
d.downloadToFile(*obj.Key)
}
return true
}
func (d *downloader) downloadToFile(key string) {
// Create the directories in the path
file := filepath.Join(d.dir, key)
if err := os.MkdirAll(filepath.Dir(file), 0775); err != nil {
panic(err)
}
fmt.Printf("Downloading " + key)
// Setup the local file
fd, err := os.Create(file)
if err != nil {
panic(err)
}
defer fd.Close()
// Download the file using the AWS SDK
fmt.Printf("Downloading s3://%s/%s to %s...\n", d.bucket, key, file)
params := &s3.GetObjectInput{Bucket: &d.bucket, Key: &key}
d.Download(fd, params)
}
The script listed the objects in the bucket very well, but did not download files into my local file system. And the script did not run into any exception.
Any idea why?
Thanks!