From cd08ebd325326996582ef02cc628caef1829c207 Mon Sep 17 00:00:00 2001 From: Rami <72725910+ramikg@users.noreply.github.com> Date: Fri, 28 Jul 2023 12:19:25 +0300 Subject: [PATCH 1/4] Remove executable bits in file open/creation --- azure_storage_client.go | 4 ++-- doc.go | 2 +- encrypt_util.go | 4 ++-- encrypt_util_test.go | 14 +++++++------- file_transfer_agent_test.go | 2 +- file_util.go | 4 ++-- gcs_storage_client.go | 4 ++-- local_storage_client.go | 4 ++-- local_storage_client_test.go | 6 +++--- ocsp.go | 2 +- ocsp_test.go | 2 +- put_get_test.go | 14 +++++++------- put_get_user_stage_test.go | 2 +- put_get_with_aws_test.go | 6 +++--- s3_storage_client.go | 2 +- secure_storage_manager_test.go | 2 +- 16 files changed, 37 insertions(+), 37 deletions(-) diff --git a/azure_storage_client.go b/azure_storage_client.go index 9a1fc3e49..be937e6ef 100644 --- a/azure_storage_client.go +++ b/azure_storage_client.go @@ -204,7 +204,7 @@ func (util *snowflakeAzureClient) uploadFile( }) } else { var f *os.File - f, err = os.OpenFile(dataFile, os.O_RDONLY, os.ModePerm) + f, err = os.OpenFile(dataFile, os.O_RDONLY, 0666) if err != nil { return err } @@ -273,7 +273,7 @@ func (util *snowflakeAzureClient) nativeDownloadFile( if meta.mockAzureClient != nil { blobClient = meta.mockAzureClient } - f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, os.ModePerm) + f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return err } diff --git a/doc.go b/doc.go index 035ed6052..f8fe39bb4 100644 --- a/doc.go +++ b/doc.go @@ -889,7 +889,7 @@ both an escape character and as a separator in path names. To send information from a stream (rather than a file) use code similar to the code below. (The ReplaceAll() function is needed on Windows to handle backslashes in the path to the file.) - fileStream, _ := os.OpenFile(fname, os.O_RDONLY, os.ModePerm) + fileStream, _ := os.OpenFile(fname, os.O_RDONLY, 0666) defer func() { if fileStream != nil { fileStream.Close() diff --git a/encrypt_util.go b/encrypt_util.go index cf4ca1528..286677fd9 100644 --- a/encrypt_util.go +++ b/encrypt_util.go @@ -176,7 +176,7 @@ func encryptFile( if err != nil { return nil, "", err } - infile, err := os.OpenFile(filename, os.O_CREATE|os.O_RDONLY, os.ModePerm) + infile, err := os.OpenFile(filename, os.O_CREATE|os.O_RDONLY, 0666) if err != nil { return nil, "", err } @@ -232,7 +232,7 @@ func decryptFile( return "", err } defer tmpOutputFile.Close() - infile, err := os.OpenFile(filename, os.O_RDONLY, os.ModePerm) + infile, err := os.OpenFile(filename, os.O_RDONLY, 0666) if err != nil { return "", err } diff --git a/encrypt_util_test.go b/encrypt_util_test.go index d7ec48814..08daff34d 100644 --- a/encrypt_util_test.go +++ b/encrypt_util_test.go @@ -33,7 +33,7 @@ func TestEncryptDecryptFile(t *testing.T) { data := "test data" inputFile := "test_encrypt_decrypt_file" - fd, err := os.OpenFile(inputFile, os.O_CREATE|os.O_WRONLY, os.ModePerm) + fd, err := os.OpenFile(inputFile, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { t.Error(err) } @@ -54,7 +54,7 @@ func TestEncryptDecryptFile(t *testing.T) { } defer os.Remove(decryptedFile) - fd, err = os.OpenFile(decryptedFile, os.O_RDONLY, os.ModePerm) + fd, err = os.OpenFile(decryptedFile, os.O_RDONLY, 0666) if err != nil { t.Error(err) } @@ -144,7 +144,7 @@ func encryptDecryptFile(t *testing.T, encMat snowflakeFileEncryption, expected i defer os.Remove(decryptedFile) cnt := 0 - fd, err := os.OpenFile(decryptedFile, os.O_RDONLY, os.ModePerm) + fd, err := os.OpenFile(decryptedFile, os.O_RDONLY, 0666) if err != nil { t.Error(err) } @@ -168,7 +168,7 @@ func generateKLinesOfNByteRows(numLines int, numBytes int, tmpDir string) (strin } } fname := path.Join(tmpDir, "file"+strconv.FormatInt(int64(numLines*numBytes), 10)) - f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, os.ModePerm) + f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return "", err } @@ -191,7 +191,7 @@ func generateKLinesOfNFiles(k int, n int, compress bool, tmpDir string) (string, } for i := 0; i < n; i++ { fname := path.Join(tmpDir, "file"+strconv.FormatInt(int64(i), 10)) - f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, os.ModePerm) + f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return "", err } @@ -233,12 +233,12 @@ func generateKLinesOfNFiles(k int, n int, compress bool, tmpDir string) (string, io.ReadAll(gzipErr) gzipCmd.Wait() } else { - fOut, err := os.OpenFile(fname+".gz", os.O_CREATE|os.O_WRONLY, os.ModePerm) + fOut, err := os.OpenFile(fname+".gz", os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return "", err } w := gzip.NewWriter(fOut) - fIn, err := os.OpenFile(fname, os.O_RDONLY, os.ModePerm) + fIn, err := os.OpenFile(fname, os.O_RDONLY, 0666) if err != nil { return "", err } diff --git a/file_transfer_agent_test.go b/file_transfer_agent_test.go index 294407539..73ecccd1c 100644 --- a/file_transfer_agent_test.go +++ b/file_transfer_agent_test.go @@ -62,7 +62,7 @@ func TestUnitDownloadWithInvalidLocalPath(t *testing.T) { } defer os.RemoveAll(tmpDir) testData := filepath.Join(tmpDir, "data.txt") - f, err := os.OpenFile(testData, os.O_CREATE|os.O_RDWR, os.ModePerm) + f, err := os.OpenFile(testData, os.O_CREATE|os.O_RDWR, 0666) if err != nil { t.Error(err) } diff --git a/file_util.go b/file_util.go index 4a0b08dd3..8b376a667 100644 --- a/file_util.go +++ b/file_util.go @@ -39,12 +39,12 @@ func (util *snowflakeFileUtil) compressFileWithGzip(fileName string, tmpDir stri basename := baseName(fileName) gzipFileName := filepath.Join(tmpDir, basename+"_c.gz") - fr, err := os.OpenFile(fileName, os.O_RDONLY, os.ModePerm) + fr, err := os.OpenFile(fileName, os.O_RDONLY, 0666) if err != nil { return "", -1, err } defer fr.Close() - fw, err := os.OpenFile(gzipFileName, os.O_WRONLY|os.O_CREATE, os.ModePerm) + fw, err := os.OpenFile(gzipFileName, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { return "", -1, err } diff --git a/gcs_storage_client.go b/gcs_storage_client.go index dbd674bcc..0b9768a5c 100644 --- a/gcs_storage_client.go +++ b/gcs_storage_client.go @@ -196,7 +196,7 @@ func (util *snowflakeGcsClient) uploadFile( uploadSrc = meta.realSrcStream } } else { - uploadSrc, err = os.OpenFile(dataFile, os.O_RDONLY, os.ModePerm) + uploadSrc, err = os.OpenFile(dataFile, os.O_RDONLY, 0666) if err != nil { return err } @@ -314,7 +314,7 @@ func (util *snowflakeGcsClient) nativeDownloadFile( return meta.lastError } - f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, os.ModePerm) + f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return err } diff --git a/local_storage_client.go b/local_storage_client.go index 883b8e3fc..69b4476b8 100644 --- a/local_storage_client.go +++ b/local_storage_client.go @@ -47,7 +47,7 @@ func (util *localUtil) uploadOneFileWithRetry(meta *fileMetadata) error { return nil } } - output, err := os.OpenFile(filepath.Join(user, meta.dstFileName), os.O_CREATE|os.O_WRONLY, os.ModePerm) + output, err := os.OpenFile(filepath.Join(user, meta.dstFileName), os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return err } @@ -100,7 +100,7 @@ func (util *localUtil) downloadOneFile(meta *fileMetadata) error { if err != nil { return err } - if err = os.WriteFile(fullDstFileName, data, os.ModePerm); err != nil { + if err = os.WriteFile(fullDstFileName, data, 0666); err != nil { return err } fi, err := os.Stat(fullDstFileName) diff --git a/local_storage_client_test.go b/local_storage_client_test.go index 1141c7060..896b94ac2 100644 --- a/local_storage_client_test.go +++ b/local_storage_client_test.go @@ -25,7 +25,7 @@ func TestLocalUpload(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err := os.WriteFile(fname, b.Bytes(), os.ModePerm); err != nil { + if err := os.WriteFile(fname, b.Bytes(), 0666); err != nil { t.Fatal("could not write to gzip file") } putDir, err := os.MkdirTemp("", "put") @@ -73,7 +73,7 @@ func TestLocalUpload(t *testing.T) { if uploadMeta.resStatus != skipped { t.Fatal("overwrite is false. should have skipped") } - fileStream, _ := os.OpenFile(fname, os.O_RDONLY, os.ModePerm) + fileStream, _ := os.OpenFile(fname, os.O_RDONLY, 0666) ctx := WithFileStream(context.Background(), fileStream) uploadMeta.srcStream = getFileStream(ctx) @@ -116,7 +116,7 @@ func TestDownloadLocalFile(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err := os.WriteFile(fname, b.Bytes(), os.ModePerm); err != nil { + if err := os.WriteFile(fname, b.Bytes(), 0666); err != nil { t.Fatal("could not write to gzip file") } putDir, err := os.MkdirTemp("", "put") diff --git a/ocsp.go b/ocsp.go index 8894a3434..d575d2292 100644 --- a/ocsp.go +++ b/ocsp.go @@ -723,7 +723,7 @@ func initOCSPCache() { ocspResponseCacheLock = &sync.RWMutex{} logger.Infof("reading OCSP Response cache file. %v\n", cacheFileName) - f, err := os.OpenFile(cacheFileName, os.O_CREATE|os.O_RDONLY, os.ModePerm) + f, err := os.OpenFile(cacheFileName, os.O_CREATE|os.O_RDONLY, 0666) if err != nil { logger.Debugf("failed to open. Ignored. %v\n", err) return diff --git a/ocsp_test.go b/ocsp_test.go index e1989568e..12a147dfd 100644 --- a/ocsp_test.go +++ b/ocsp_test.go @@ -515,7 +515,7 @@ func TestInitOCSPCacheFileCreation(t *testing.T) { defer func() { src, _ = os.Open(tmpFileName) defer src.Close() - dst, _ = os.OpenFile(srcFileName, os.O_WRONLY, os.ModePerm) + dst, _ = os.OpenFile(srcFileName, os.O_WRONLY, 0666) defer dst.Close() // copy temporary file contents back to original file if _, err = io.Copy(dst, src); err != nil { diff --git a/put_get_test.go b/put_get_test.go index d1666dc9a..0247447a7 100644 --- a/put_get_test.go +++ b/put_get_test.go @@ -41,7 +41,7 @@ func TestPutError(t *testing.T) { defer os.RemoveAll(tmpDir) file1 := filepath.Join(tmpDir, "file1") remoteLocation := filepath.Join(tmpDir, "remote_loc") - f, err := os.OpenFile(file1, os.O_CREATE|os.O_WRONLY, os.ModePerm) + f, err := os.OpenFile(file1, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { t.Error(err) } @@ -250,7 +250,7 @@ func TestPutWithAutoCompressFalse(t *testing.T) { } defer os.RemoveAll(tmpDir) testData := filepath.Join(tmpDir, "data.txt") - f, err := os.OpenFile(testData, os.O_CREATE|os.O_WRONLY, os.ModePerm) + f, err := os.OpenFile(testData, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { t.Error(err) } @@ -291,7 +291,7 @@ func TestPutOverwrite(t *testing.T) { } defer os.RemoveAll(tmpDir) testData := filepath.Join(tmpDir, "data.txt") - f, err := os.OpenFile(testData, os.O_CREATE|os.O_RDWR, os.ModePerm) + f, err := os.OpenFile(testData, os.O_CREATE|os.O_RDWR, 0666) if err != nil { t.Error(err) } @@ -392,14 +392,14 @@ func testPutGet(t *testing.T, isStream bool) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err = os.WriteFile(fname, b.Bytes(), os.ModePerm); err != nil { + if err = os.WriteFile(fname, b.Bytes(), 0666); err != nil { t.Fatal("could not write to gzip file") } runTests(t, dsn, func(dbt *DBTest) { dbt.mustExec("create or replace table " + tableName + " (a int, b string)") - fileStream, err := os.OpenFile(fname, os.O_RDONLY, os.ModePerm) + fileStream, err := os.OpenFile(fname, os.O_RDONLY, 0666) if err != nil { t.Error(err) } @@ -517,7 +517,7 @@ func TestPutGetGcsDownscopedCredential(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err = os.WriteFile(fname, b.Bytes(), os.ModePerm); err != nil { + if err = os.WriteFile(fname, b.Bytes(), 0666); err != nil { t.Fatal("could not write to gzip file") } @@ -525,7 +525,7 @@ func TestPutGetGcsDownscopedCredential(t *testing.T) { runTests(t, dsn, func(dbt *DBTest) { dbt.mustExec("create or replace table " + tableName + " (a int, b string)") - fileStream, err := os.OpenFile(fname, os.O_RDONLY, os.ModePerm) + fileStream, err := os.OpenFile(fname, os.O_RDONLY, 0666) if err != nil { t.Error(err) } diff --git a/put_get_user_stage_test.go b/put_get_user_stage_test.go index aeef80c19..a1934caab 100644 --- a/put_get_user_stage_test.go +++ b/put_get_user_stage_test.go @@ -69,7 +69,7 @@ func putGetUserStage(t *testing.T, tmpDir string, numberOfFiles int, numberOfLin dbt.mustExec("rm @" + stageName) var fs *os.File if isStream { - fs, _ = os.OpenFile(files, os.O_RDONLY, os.ModePerm) + fs, _ = os.OpenFile(files, os.O_RDONLY, 0666) dbt.mustExecContext(WithFileStream(context.Background(), fs), fmt.Sprintf("put 'file://%v' @%v", strings.ReplaceAll( files, "\\", "\\\\"), stageName)) diff --git a/put_get_with_aws_test.go b/put_get_with_aws_test.go index 1a43b2c67..9a55ebab0 100644 --- a/put_get_with_aws_test.go +++ b/put_get_with_aws_test.go @@ -102,7 +102,7 @@ func TestPutWithInvalidToken(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err := os.WriteFile(fname, b.Bytes(), os.ModePerm); err != nil { + if err := os.WriteFile(fname, b.Bytes(), 0666); err != nil { t.Fatal("could not write to gzip file") } @@ -214,7 +214,7 @@ func TestPretendToPutButList(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err := os.WriteFile(fname, b.Bytes(), os.ModePerm); err != nil { + if err := os.WriteFile(fname, b.Bytes(), 0666); err != nil { t.Fatal("could not write to gzip file") } @@ -283,7 +283,7 @@ func TestPutGetAWSStage(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err = os.WriteFile(fname, b.Bytes(), os.ModePerm); err != nil { + if err = os.WriteFile(fname, b.Bytes(), 0666); err != nil { t.Fatal("could not write to gzip file") } diff --git a/s3_storage_client.go b/s3_storage_client.go index 28376d9e2..cb4987e7c 100644 --- a/s3_storage_client.go +++ b/s3_storage_client.go @@ -217,7 +217,7 @@ func (util *snowflakeS3Client) nativeDownloadFile( } } - f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, os.ModePerm) + f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return err } diff --git a/secure_storage_manager_test.go b/secure_storage_manager_test.go index dbcaef2b7..7782612f0 100644 --- a/secure_storage_manager_test.go +++ b/secure_storage_manager_test.go @@ -122,7 +122,7 @@ func TestCreateCredentialCache(t *testing.T) { // cleanup src, _ = os.Open(tmpFileName) defer src.Close() - dst, _ = os.OpenFile(srcFileName, os.O_WRONLY, os.ModePerm) + dst, _ = os.OpenFile(srcFileName, os.O_WRONLY, 0666) defer dst.Close() // copy temporary file contents back to original file if _, err = io.Copy(dst, src); err != nil { From d9cae9679688fa4787f8b13de817b72be95d6d28 Mon Sep 17 00:00:00 2001 From: ramikg <72725910+ramikg@users.noreply.github.com> Date: Tue, 1 Aug 2023 13:16:39 +0300 Subject: [PATCH 2/4] Add `readWriteFileMode` constant --- azure_storage_client.go | 4 ++-- encrypt_util.go | 4 ++-- encrypt_util_test.go | 14 +++++++------- file_transfer_agent_test.go | 2 +- file_util.go | 7 ++++--- gcs_storage_client.go | 4 ++-- local_storage_client.go | 4 ++-- local_storage_client_test.go | 6 +++--- ocsp.go | 2 +- ocsp_test.go | 2 +- put_get_test.go | 14 +++++++------- put_get_user_stage_test.go | 2 +- put_get_with_aws_test.go | 6 +++--- s3_storage_client.go | 2 +- secure_storage_manager_test.go | 2 +- 15 files changed, 38 insertions(+), 37 deletions(-) diff --git a/azure_storage_client.go b/azure_storage_client.go index be937e6ef..f5c270a63 100644 --- a/azure_storage_client.go +++ b/azure_storage_client.go @@ -204,7 +204,7 @@ func (util *snowflakeAzureClient) uploadFile( }) } else { var f *os.File - f, err = os.OpenFile(dataFile, os.O_RDONLY, 0666) + f, err = os.OpenFile(dataFile, os.O_RDONLY, readWriteFileMode) if err != nil { return err } @@ -273,7 +273,7 @@ func (util *snowflakeAzureClient) nativeDownloadFile( if meta.mockAzureClient != nil { blobClient = meta.mockAzureClient } - f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, 0666) + f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, readWriteFileMode) if err != nil { return err } diff --git a/encrypt_util.go b/encrypt_util.go index 286677fd9..b6cedcdff 100644 --- a/encrypt_util.go +++ b/encrypt_util.go @@ -176,7 +176,7 @@ func encryptFile( if err != nil { return nil, "", err } - infile, err := os.OpenFile(filename, os.O_CREATE|os.O_RDONLY, 0666) + infile, err := os.OpenFile(filename, os.O_CREATE|os.O_RDONLY, readWriteFileMode) if err != nil { return nil, "", err } @@ -232,7 +232,7 @@ func decryptFile( return "", err } defer tmpOutputFile.Close() - infile, err := os.OpenFile(filename, os.O_RDONLY, 0666) + infile, err := os.OpenFile(filename, os.O_RDONLY, readWriteFileMode) if err != nil { return "", err } diff --git a/encrypt_util_test.go b/encrypt_util_test.go index 08daff34d..8ed977f90 100644 --- a/encrypt_util_test.go +++ b/encrypt_util_test.go @@ -33,7 +33,7 @@ func TestEncryptDecryptFile(t *testing.T) { data := "test data" inputFile := "test_encrypt_decrypt_file" - fd, err := os.OpenFile(inputFile, os.O_CREATE|os.O_WRONLY, 0666) + fd, err := os.OpenFile(inputFile, os.O_CREATE|os.O_WRONLY, readWriteFileMode) if err != nil { t.Error(err) } @@ -54,7 +54,7 @@ func TestEncryptDecryptFile(t *testing.T) { } defer os.Remove(decryptedFile) - fd, err = os.OpenFile(decryptedFile, os.O_RDONLY, 0666) + fd, err = os.OpenFile(decryptedFile, os.O_RDONLY, readWriteFileMode) if err != nil { t.Error(err) } @@ -144,7 +144,7 @@ func encryptDecryptFile(t *testing.T, encMat snowflakeFileEncryption, expected i defer os.Remove(decryptedFile) cnt := 0 - fd, err := os.OpenFile(decryptedFile, os.O_RDONLY, 0666) + fd, err := os.OpenFile(decryptedFile, os.O_RDONLY, readWriteFileMode) if err != nil { t.Error(err) } @@ -168,7 +168,7 @@ func generateKLinesOfNByteRows(numLines int, numBytes int, tmpDir string) (strin } } fname := path.Join(tmpDir, "file"+strconv.FormatInt(int64(numLines*numBytes), 10)) - f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, 0666) + f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, readWriteFileMode) if err != nil { return "", err } @@ -191,7 +191,7 @@ func generateKLinesOfNFiles(k int, n int, compress bool, tmpDir string) (string, } for i := 0; i < n; i++ { fname := path.Join(tmpDir, "file"+strconv.FormatInt(int64(i), 10)) - f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, 0666) + f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, readWriteFileMode) if err != nil { return "", err } @@ -233,12 +233,12 @@ func generateKLinesOfNFiles(k int, n int, compress bool, tmpDir string) (string, io.ReadAll(gzipErr) gzipCmd.Wait() } else { - fOut, err := os.OpenFile(fname+".gz", os.O_CREATE|os.O_WRONLY, 0666) + fOut, err := os.OpenFile(fname+".gz", os.O_CREATE|os.O_WRONLY, readWriteFileMode) if err != nil { return "", err } w := gzip.NewWriter(fOut) - fIn, err := os.OpenFile(fname, os.O_RDONLY, 0666) + fIn, err := os.OpenFile(fname, os.O_RDONLY, readWriteFileMode) if err != nil { return "", err } diff --git a/file_transfer_agent_test.go b/file_transfer_agent_test.go index 73ecccd1c..f48ba6268 100644 --- a/file_transfer_agent_test.go +++ b/file_transfer_agent_test.go @@ -62,7 +62,7 @@ func TestUnitDownloadWithInvalidLocalPath(t *testing.T) { } defer os.RemoveAll(tmpDir) testData := filepath.Join(tmpDir, "data.txt") - f, err := os.OpenFile(testData, os.O_CREATE|os.O_RDWR, 0666) + f, err := os.OpenFile(testData, os.O_CREATE|os.O_RDWR, readWriteFileMode) if err != nil { t.Error(err) } diff --git a/file_util.go b/file_util.go index 8b376a667..cf32fdc97 100644 --- a/file_util.go +++ b/file_util.go @@ -19,7 +19,8 @@ type snowflakeFileUtil struct { } const ( - fileChunkSize = 16 * 4 * 1024 + fileChunkSize = 16 * 4 * 1024 + readWriteFileMode os.FileMode = 0666 ) func (util *snowflakeFileUtil) compressFileWithGzipFromStream(srcStream **bytes.Buffer) (*bytes.Buffer, int, error) { @@ -39,12 +40,12 @@ func (util *snowflakeFileUtil) compressFileWithGzip(fileName string, tmpDir stri basename := baseName(fileName) gzipFileName := filepath.Join(tmpDir, basename+"_c.gz") - fr, err := os.OpenFile(fileName, os.O_RDONLY, 0666) + fr, err := os.OpenFile(fileName, os.O_RDONLY, readWriteFileMode) if err != nil { return "", -1, err } defer fr.Close() - fw, err := os.OpenFile(gzipFileName, os.O_WRONLY|os.O_CREATE, 0666) + fw, err := os.OpenFile(gzipFileName, os.O_WRONLY|os.O_CREATE, readWriteFileMode) if err != nil { return "", -1, err } diff --git a/gcs_storage_client.go b/gcs_storage_client.go index 0b9768a5c..078091b25 100644 --- a/gcs_storage_client.go +++ b/gcs_storage_client.go @@ -196,7 +196,7 @@ func (util *snowflakeGcsClient) uploadFile( uploadSrc = meta.realSrcStream } } else { - uploadSrc, err = os.OpenFile(dataFile, os.O_RDONLY, 0666) + uploadSrc, err = os.OpenFile(dataFile, os.O_RDONLY, readWriteFileMode) if err != nil { return err } @@ -314,7 +314,7 @@ func (util *snowflakeGcsClient) nativeDownloadFile( return meta.lastError } - f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, 0666) + f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, readWriteFileMode) if err != nil { return err } diff --git a/local_storage_client.go b/local_storage_client.go index 69b4476b8..2ae072b63 100644 --- a/local_storage_client.go +++ b/local_storage_client.go @@ -47,7 +47,7 @@ func (util *localUtil) uploadOneFileWithRetry(meta *fileMetadata) error { return nil } } - output, err := os.OpenFile(filepath.Join(user, meta.dstFileName), os.O_CREATE|os.O_WRONLY, 0666) + output, err := os.OpenFile(filepath.Join(user, meta.dstFileName), os.O_CREATE|os.O_WRONLY, readWriteFileMode) if err != nil { return err } @@ -100,7 +100,7 @@ func (util *localUtil) downloadOneFile(meta *fileMetadata) error { if err != nil { return err } - if err = os.WriteFile(fullDstFileName, data, 0666); err != nil { + if err = os.WriteFile(fullDstFileName, data, readWriteFileMode); err != nil { return err } fi, err := os.Stat(fullDstFileName) diff --git a/local_storage_client_test.go b/local_storage_client_test.go index 896b94ac2..114f49982 100644 --- a/local_storage_client_test.go +++ b/local_storage_client_test.go @@ -25,7 +25,7 @@ func TestLocalUpload(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err := os.WriteFile(fname, b.Bytes(), 0666); err != nil { + if err := os.WriteFile(fname, b.Bytes(), readWriteFileMode); err != nil { t.Fatal("could not write to gzip file") } putDir, err := os.MkdirTemp("", "put") @@ -73,7 +73,7 @@ func TestLocalUpload(t *testing.T) { if uploadMeta.resStatus != skipped { t.Fatal("overwrite is false. should have skipped") } - fileStream, _ := os.OpenFile(fname, os.O_RDONLY, 0666) + fileStream, _ := os.OpenFile(fname, os.O_RDONLY, readWriteFileMode) ctx := WithFileStream(context.Background(), fileStream) uploadMeta.srcStream = getFileStream(ctx) @@ -116,7 +116,7 @@ func TestDownloadLocalFile(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err := os.WriteFile(fname, b.Bytes(), 0666); err != nil { + if err := os.WriteFile(fname, b.Bytes(), readWriteFileMode); err != nil { t.Fatal("could not write to gzip file") } putDir, err := os.MkdirTemp("", "put") diff --git a/ocsp.go b/ocsp.go index d575d2292..f0a30b995 100644 --- a/ocsp.go +++ b/ocsp.go @@ -723,7 +723,7 @@ func initOCSPCache() { ocspResponseCacheLock = &sync.RWMutex{} logger.Infof("reading OCSP Response cache file. %v\n", cacheFileName) - f, err := os.OpenFile(cacheFileName, os.O_CREATE|os.O_RDONLY, 0666) + f, err := os.OpenFile(cacheFileName, os.O_CREATE|os.O_RDONLY, readWriteFileMode) if err != nil { logger.Debugf("failed to open. Ignored. %v\n", err) return diff --git a/ocsp_test.go b/ocsp_test.go index 12a147dfd..678d1eadd 100644 --- a/ocsp_test.go +++ b/ocsp_test.go @@ -515,7 +515,7 @@ func TestInitOCSPCacheFileCreation(t *testing.T) { defer func() { src, _ = os.Open(tmpFileName) defer src.Close() - dst, _ = os.OpenFile(srcFileName, os.O_WRONLY, 0666) + dst, _ = os.OpenFile(srcFileName, os.O_WRONLY, readWriteFileMode) defer dst.Close() // copy temporary file contents back to original file if _, err = io.Copy(dst, src); err != nil { diff --git a/put_get_test.go b/put_get_test.go index 0247447a7..259abe178 100644 --- a/put_get_test.go +++ b/put_get_test.go @@ -41,7 +41,7 @@ func TestPutError(t *testing.T) { defer os.RemoveAll(tmpDir) file1 := filepath.Join(tmpDir, "file1") remoteLocation := filepath.Join(tmpDir, "remote_loc") - f, err := os.OpenFile(file1, os.O_CREATE|os.O_WRONLY, 0666) + f, err := os.OpenFile(file1, os.O_CREATE|os.O_WRONLY, readWriteFileMode) if err != nil { t.Error(err) } @@ -250,7 +250,7 @@ func TestPutWithAutoCompressFalse(t *testing.T) { } defer os.RemoveAll(tmpDir) testData := filepath.Join(tmpDir, "data.txt") - f, err := os.OpenFile(testData, os.O_CREATE|os.O_WRONLY, 0666) + f, err := os.OpenFile(testData, os.O_CREATE|os.O_WRONLY, readWriteFileMode) if err != nil { t.Error(err) } @@ -291,7 +291,7 @@ func TestPutOverwrite(t *testing.T) { } defer os.RemoveAll(tmpDir) testData := filepath.Join(tmpDir, "data.txt") - f, err := os.OpenFile(testData, os.O_CREATE|os.O_RDWR, 0666) + f, err := os.OpenFile(testData, os.O_CREATE|os.O_RDWR, readWriteFileMode) if err != nil { t.Error(err) } @@ -392,14 +392,14 @@ func testPutGet(t *testing.T, isStream bool) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err = os.WriteFile(fname, b.Bytes(), 0666); err != nil { + if err = os.WriteFile(fname, b.Bytes(), readWriteFileMode); err != nil { t.Fatal("could not write to gzip file") } runTests(t, dsn, func(dbt *DBTest) { dbt.mustExec("create or replace table " + tableName + " (a int, b string)") - fileStream, err := os.OpenFile(fname, os.O_RDONLY, 0666) + fileStream, err := os.OpenFile(fname, os.O_RDONLY, readWriteFileMode) if err != nil { t.Error(err) } @@ -517,7 +517,7 @@ func TestPutGetGcsDownscopedCredential(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err = os.WriteFile(fname, b.Bytes(), 0666); err != nil { + if err = os.WriteFile(fname, b.Bytes(), readWriteFileMode); err != nil { t.Fatal("could not write to gzip file") } @@ -525,7 +525,7 @@ func TestPutGetGcsDownscopedCredential(t *testing.T) { runTests(t, dsn, func(dbt *DBTest) { dbt.mustExec("create or replace table " + tableName + " (a int, b string)") - fileStream, err := os.OpenFile(fname, os.O_RDONLY, 0666) + fileStream, err := os.OpenFile(fname, os.O_RDONLY, readWriteFileMode) if err != nil { t.Error(err) } diff --git a/put_get_user_stage_test.go b/put_get_user_stage_test.go index a1934caab..d8f6a0c88 100644 --- a/put_get_user_stage_test.go +++ b/put_get_user_stage_test.go @@ -69,7 +69,7 @@ func putGetUserStage(t *testing.T, tmpDir string, numberOfFiles int, numberOfLin dbt.mustExec("rm @" + stageName) var fs *os.File if isStream { - fs, _ = os.OpenFile(files, os.O_RDONLY, 0666) + fs, _ = os.OpenFile(files, os.O_RDONLY, readWriteFileMode) dbt.mustExecContext(WithFileStream(context.Background(), fs), fmt.Sprintf("put 'file://%v' @%v", strings.ReplaceAll( files, "\\", "\\\\"), stageName)) diff --git a/put_get_with_aws_test.go b/put_get_with_aws_test.go index 9a55ebab0..8b352cee7 100644 --- a/put_get_with_aws_test.go +++ b/put_get_with_aws_test.go @@ -102,7 +102,7 @@ func TestPutWithInvalidToken(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err := os.WriteFile(fname, b.Bytes(), 0666); err != nil { + if err := os.WriteFile(fname, b.Bytes(), readWriteFileMode); err != nil { t.Fatal("could not write to gzip file") } @@ -214,7 +214,7 @@ func TestPretendToPutButList(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err := os.WriteFile(fname, b.Bytes(), 0666); err != nil { + if err := os.WriteFile(fname, b.Bytes(), readWriteFileMode); err != nil { t.Fatal("could not write to gzip file") } @@ -283,7 +283,7 @@ func TestPutGetAWSStage(t *testing.T) { gzw := gzip.NewWriter(&b) gzw.Write([]byte(originalContents)) gzw.Close() - if err = os.WriteFile(fname, b.Bytes(), 0666); err != nil { + if err = os.WriteFile(fname, b.Bytes(), readWriteFileMode); err != nil { t.Fatal("could not write to gzip file") } diff --git a/s3_storage_client.go b/s3_storage_client.go index cb4987e7c..ed3bca59a 100644 --- a/s3_storage_client.go +++ b/s3_storage_client.go @@ -217,7 +217,7 @@ func (util *snowflakeS3Client) nativeDownloadFile( } } - f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, 0666) + f, err := os.OpenFile(fullDstFileName, os.O_CREATE|os.O_WRONLY, readWriteFileMode) if err != nil { return err } diff --git a/secure_storage_manager_test.go b/secure_storage_manager_test.go index 7782612f0..252543d27 100644 --- a/secure_storage_manager_test.go +++ b/secure_storage_manager_test.go @@ -122,7 +122,7 @@ func TestCreateCredentialCache(t *testing.T) { // cleanup src, _ = os.Open(tmpFileName) defer src.Close() - dst, _ = os.OpenFile(srcFileName, os.O_WRONLY, 0666) + dst, _ = os.OpenFile(srcFileName, os.O_WRONLY, readWriteFileMode) defer dst.Close() // copy temporary file contents back to original file if _, err = io.Copy(dst, src); err != nil { From 98df002eee06e706c79192b1d80c63ea5f9bee88 Mon Sep 17 00:00:00 2001 From: ramikg <72725910+ramikg@users.noreply.github.com> Date: Tue, 1 Aug 2023 13:22:57 +0300 Subject: [PATCH 3/4] Replace `os.OpenFile` with `os.Open` where equivalent --- azure_storage_client.go | 2 +- doc.go | 2 +- encrypt_util.go | 2 +- encrypt_util_test.go | 6 +++--- file_util.go | 2 +- gcs_storage_client.go | 2 +- local_storage_client_test.go | 2 +- put_get_test.go | 4 ++-- put_get_user_stage_test.go | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/azure_storage_client.go b/azure_storage_client.go index f5c270a63..0db3a4cb3 100644 --- a/azure_storage_client.go +++ b/azure_storage_client.go @@ -204,7 +204,7 @@ func (util *snowflakeAzureClient) uploadFile( }) } else { var f *os.File - f, err = os.OpenFile(dataFile, os.O_RDONLY, readWriteFileMode) + f, err = os.Open(dataFile) if err != nil { return err } diff --git a/doc.go b/doc.go index f8fe39bb4..d6ab5a8c0 100644 --- a/doc.go +++ b/doc.go @@ -889,7 +889,7 @@ both an escape character and as a separator in path names. To send information from a stream (rather than a file) use code similar to the code below. (The ReplaceAll() function is needed on Windows to handle backslashes in the path to the file.) - fileStream, _ := os.OpenFile(fname, os.O_RDONLY, 0666) + fileStream, _ := os.Open(fname) defer func() { if fileStream != nil { fileStream.Close() diff --git a/encrypt_util.go b/encrypt_util.go index b6cedcdff..435ecbafc 100644 --- a/encrypt_util.go +++ b/encrypt_util.go @@ -232,7 +232,7 @@ func decryptFile( return "", err } defer tmpOutputFile.Close() - infile, err := os.OpenFile(filename, os.O_RDONLY, readWriteFileMode) + infile, err := os.Open(filename) if err != nil { return "", err } diff --git a/encrypt_util_test.go b/encrypt_util_test.go index 8ed977f90..1480a3ab9 100644 --- a/encrypt_util_test.go +++ b/encrypt_util_test.go @@ -54,7 +54,7 @@ func TestEncryptDecryptFile(t *testing.T) { } defer os.Remove(decryptedFile) - fd, err = os.OpenFile(decryptedFile, os.O_RDONLY, readWriteFileMode) + fd, err = os.Open(decryptedFile) if err != nil { t.Error(err) } @@ -144,7 +144,7 @@ func encryptDecryptFile(t *testing.T, encMat snowflakeFileEncryption, expected i defer os.Remove(decryptedFile) cnt := 0 - fd, err := os.OpenFile(decryptedFile, os.O_RDONLY, readWriteFileMode) + fd, err := os.Open(decryptedFile) if err != nil { t.Error(err) } @@ -238,7 +238,7 @@ func generateKLinesOfNFiles(k int, n int, compress bool, tmpDir string) (string, return "", err } w := gzip.NewWriter(fOut) - fIn, err := os.OpenFile(fname, os.O_RDONLY, readWriteFileMode) + fIn, err := os.Open(fname) if err != nil { return "", err } diff --git a/file_util.go b/file_util.go index cf32fdc97..13740bb89 100644 --- a/file_util.go +++ b/file_util.go @@ -40,7 +40,7 @@ func (util *snowflakeFileUtil) compressFileWithGzip(fileName string, tmpDir stri basename := baseName(fileName) gzipFileName := filepath.Join(tmpDir, basename+"_c.gz") - fr, err := os.OpenFile(fileName, os.O_RDONLY, readWriteFileMode) + fr, err := os.Open(fileName) if err != nil { return "", -1, err } diff --git a/gcs_storage_client.go b/gcs_storage_client.go index 078091b25..5d9dac9a8 100644 --- a/gcs_storage_client.go +++ b/gcs_storage_client.go @@ -196,7 +196,7 @@ func (util *snowflakeGcsClient) uploadFile( uploadSrc = meta.realSrcStream } } else { - uploadSrc, err = os.OpenFile(dataFile, os.O_RDONLY, readWriteFileMode) + uploadSrc, err = os.Open(dataFile) if err != nil { return err } diff --git a/local_storage_client_test.go b/local_storage_client_test.go index 114f49982..6b0c87f7c 100644 --- a/local_storage_client_test.go +++ b/local_storage_client_test.go @@ -73,7 +73,7 @@ func TestLocalUpload(t *testing.T) { if uploadMeta.resStatus != skipped { t.Fatal("overwrite is false. should have skipped") } - fileStream, _ := os.OpenFile(fname, os.O_RDONLY, readWriteFileMode) + fileStream, _ := os.Open(fname) ctx := WithFileStream(context.Background(), fileStream) uploadMeta.srcStream = getFileStream(ctx) diff --git a/put_get_test.go b/put_get_test.go index 259abe178..93f2a9fa8 100644 --- a/put_get_test.go +++ b/put_get_test.go @@ -399,7 +399,7 @@ func testPutGet(t *testing.T, isStream bool) { runTests(t, dsn, func(dbt *DBTest) { dbt.mustExec("create or replace table " + tableName + " (a int, b string)") - fileStream, err := os.OpenFile(fname, os.O_RDONLY, readWriteFileMode) + fileStream, err := os.Open(fname) if err != nil { t.Error(err) } @@ -525,7 +525,7 @@ func TestPutGetGcsDownscopedCredential(t *testing.T) { runTests(t, dsn, func(dbt *DBTest) { dbt.mustExec("create or replace table " + tableName + " (a int, b string)") - fileStream, err := os.OpenFile(fname, os.O_RDONLY, readWriteFileMode) + fileStream, err := os.Open(fname) if err != nil { t.Error(err) } diff --git a/put_get_user_stage_test.go b/put_get_user_stage_test.go index d8f6a0c88..69c9a1142 100644 --- a/put_get_user_stage_test.go +++ b/put_get_user_stage_test.go @@ -69,7 +69,7 @@ func putGetUserStage(t *testing.T, tmpDir string, numberOfFiles int, numberOfLin dbt.mustExec("rm @" + stageName) var fs *os.File if isStream { - fs, _ = os.OpenFile(files, os.O_RDONLY, readWriteFileMode) + fs, _ = os.Open(files) dbt.mustExecContext(WithFileStream(context.Background(), fs), fmt.Sprintf("put 'file://%v' @%v", strings.ReplaceAll( files, "\\", "\\\\"), stageName)) From f55635248ac775437a5cacf06d6a286cc677e173 Mon Sep 17 00:00:00 2001 From: ramikg <72725910+ramikg@users.noreply.github.com> Date: Tue, 1 Aug 2023 16:57:27 +0300 Subject: [PATCH 4/4] Replace `os.OpenFile` with `os.Create` in tests --- encrypt_util_test.go | 8 ++++---- file_transfer_agent_test.go | 2 +- put_get_test.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/encrypt_util_test.go b/encrypt_util_test.go index 1480a3ab9..5d0d9988b 100644 --- a/encrypt_util_test.go +++ b/encrypt_util_test.go @@ -33,7 +33,7 @@ func TestEncryptDecryptFile(t *testing.T) { data := "test data" inputFile := "test_encrypt_decrypt_file" - fd, err := os.OpenFile(inputFile, os.O_CREATE|os.O_WRONLY, readWriteFileMode) + fd, err := os.Create(inputFile) if err != nil { t.Error(err) } @@ -168,7 +168,7 @@ func generateKLinesOfNByteRows(numLines int, numBytes int, tmpDir string) (strin } } fname := path.Join(tmpDir, "file"+strconv.FormatInt(int64(numLines*numBytes), 10)) - f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, readWriteFileMode) + f, err := os.Create(fname) if err != nil { return "", err } @@ -191,7 +191,7 @@ func generateKLinesOfNFiles(k int, n int, compress bool, tmpDir string) (string, } for i := 0; i < n; i++ { fname := path.Join(tmpDir, "file"+strconv.FormatInt(int64(i), 10)) - f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, readWriteFileMode) + f, err := os.Create(fname) if err != nil { return "", err } @@ -233,7 +233,7 @@ func generateKLinesOfNFiles(k int, n int, compress bool, tmpDir string) (string, io.ReadAll(gzipErr) gzipCmd.Wait() } else { - fOut, err := os.OpenFile(fname+".gz", os.O_CREATE|os.O_WRONLY, readWriteFileMode) + fOut, err := os.Create(fname + ".gz") if err != nil { return "", err } diff --git a/file_transfer_agent_test.go b/file_transfer_agent_test.go index f48ba6268..ce94dfbe5 100644 --- a/file_transfer_agent_test.go +++ b/file_transfer_agent_test.go @@ -62,7 +62,7 @@ func TestUnitDownloadWithInvalidLocalPath(t *testing.T) { } defer os.RemoveAll(tmpDir) testData := filepath.Join(tmpDir, "data.txt") - f, err := os.OpenFile(testData, os.O_CREATE|os.O_RDWR, readWriteFileMode) + f, err := os.Create(testData) if err != nil { t.Error(err) } diff --git a/put_get_test.go b/put_get_test.go index 93f2a9fa8..0bfce1f36 100644 --- a/put_get_test.go +++ b/put_get_test.go @@ -41,7 +41,7 @@ func TestPutError(t *testing.T) { defer os.RemoveAll(tmpDir) file1 := filepath.Join(tmpDir, "file1") remoteLocation := filepath.Join(tmpDir, "remote_loc") - f, err := os.OpenFile(file1, os.O_CREATE|os.O_WRONLY, readWriteFileMode) + f, err := os.Create(file1) if err != nil { t.Error(err) } @@ -250,7 +250,7 @@ func TestPutWithAutoCompressFalse(t *testing.T) { } defer os.RemoveAll(tmpDir) testData := filepath.Join(tmpDir, "data.txt") - f, err := os.OpenFile(testData, os.O_CREATE|os.O_WRONLY, readWriteFileMode) + f, err := os.Create(testData) if err != nil { t.Error(err) } @@ -291,7 +291,7 @@ func TestPutOverwrite(t *testing.T) { } defer os.RemoveAll(tmpDir) testData := filepath.Join(tmpDir, "data.txt") - f, err := os.OpenFile(testData, os.O_CREATE|os.O_RDWR, readWriteFileMode) + f, err := os.Create(testData) if err != nil { t.Error(err) }