Skip to content
GitLab

Download Helper modifications

Closed Download Helper modifications
DownloadHelper_Retry_SSS into master
Closed Sandeep S. Shewalkar requested to merge DownloadHelper_Retry_SSS into master
Compare
and
1 file
+ 109
− 96
Preferences
File browser
Compare changes
downloadhelper/downloadHelper.go
+ 109
− 96
@@ -2,10 +2,13 @@ package downloadhelper
import (
"errors"
"path"
"path/filepath"
"strings"
"time"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/coreospackage/filehelper"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/coreospackage/securityhelper"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/coreospackage/logginghelper"
"github.com/cavaliercoder/grab"
@@ -14,6 +17,7 @@ import (
const (
EMPTY_STR = ""
HTTP_PROTOCOL = "http"
ERR_FILE_HASH = 6273839363813614546
)
//DownloadFile file from cloud
@@ -22,6 +26,7 @@ const (
// jwtToken if any
// retryCnt : how many times process should retry for download.
func DownloadFile(sourcePath, destinationPath, jwtToken string, retryCnt int) (bool, error) {
destinationPath = filehelper.CleanPath(destinationPath)
if strings.Trim(sourcePath, EMPTY_STR) == EMPTY_STR || strings.Trim(destinationPath, EMPTY_STR) == EMPTY_STR || !strings.HasPrefix(sourcePath, HTTP_PROTOCOL) {
return false, errors.New("Either Source/Destination url is empty or source url does not start with 'http'")
}
@@ -32,120 +37,128 @@ func DownloadFile(sourcePath, destinationPath, jwtToken string, retryCnt int) (b
}
for index := 1; index <= retryCnt; index++ {
err := BackupFile(destinationPath)
if err != nil {
logginghelper.LogError("GrabDownloadFile Backup Error : ", err)
return false, err
}
resp := client.Do(req)
if resp != nil && (resp.Size <= 0 || (resp.HTTPResponse != nil && resp.HTTPResponse.StatusCode == 404)) {
err := filehelper.DeleteFile(destinationPath)
if err != nil {
logginghelper.LogError("Error in deleting file \n : ", destinationPath, err)
}
if err := resp.Err(); err != nil {
logginghelper.LogError("Download failed: ", err)
RestoreBackup(destinationPath)
time.Sleep(200 * time.Millisecond)
continue
}
// t := time.NewTicker(200 * time.Millisecond)
// defer t.Stop()
// Loop:
// for {
// select {
// // case <-t.C:
// // logginghelper.LogInfo(" transferred %v / %v bytes (%.2f%%)\n",
// // resp.BytesComplete(),
// // resp.Size,
// // 100*resp.Progress())
// case <-resp.Done:
// // download is complete
// break Loop
// }
// }
// check for errors
if err := resp.Err(); err != nil {
logginghelper.LogError("Download failed: %v\n", err)
return false, errors.New("File not found on cloud")
if resp == nil || resp.HTTPResponse == nil {
logginghelper.LogError("Download failed: ", err)
RestoreBackup(destinationPath)
time.Sleep(200 * time.Millisecond)
continue
} else if resp.HTTPResponse != nil && resp.HTTPResponse.StatusCode != 200 && resp.HTTPResponse.StatusCode != 206 {
logginghelper.LogError("Download failed.")
RestoreBackup(destinationPath)
time.Sleep(200 * time.Millisecond)
continue
} else {
fileInfo, err := filehelper.FileInfo(destinationPath)
if err != nil {
logginghelper.LogError("Download failed: ", err)
RestoreBackup(destinationPath)
}
if fileInfo.Size() < FILE_SIZE {
hash, err := securityhelper.GetHashChecksumOfFile(destinationPath)
if err != nil {
logginghelper.LogError("Download failed: ", err)
RestoreBackup(destinationPath)
}
if hash == ERR_FILE_HASH {
logginghelper.LogError("Download failed: ", err)
RestoreBackup(destinationPath)
}
time.Sleep(200 * time.Millisecond)
continue
}
}
logginghelper.LogDebug("resp.HTTPResponse.StatusCode: ", resp.HTTPResponse.StatusCode)
//resp, err = grab.Get(destinationPath, sourcePath)
if resp.HTTPResponse != nil && (resp.HTTPResponse.StatusCode == 200 || resp.HTTPResponse.StatusCode == 206) {
closeErr := resp.HTTPResponse.Body.Close()
if closeErr != nil {
logginghelper.LogError("Error while closing response : ", closeErr)
}
DeleteBackup(destinationPath)
return true, nil
}
err := filehelper.DeleteFile(destinationPath)
if err != nil {
logginghelper.LogError("Error in deleting file \n : ", destinationPath, err)
return false, err
}
logginghelper.LogError("Download failed: ", err)
RestoreBackup(destinationPath)
return false, nil
}
err := filehelper.DeleteFile(destinationPath)
if err != nil {
logginghelper.LogError("Error in deleting file \n : ", destinationPath, err)
return false, err
}
logginghelper.LogError("Download failed.")
return false, errors.New("File not found on cloud")
}
// func DownloadFile(sourcePath, destinationPath, jwtToken string, retryCnt int) (bool, error) {
// if strings.Trim(sourcePath, EMPTY_STR) == EMPTY_STR || strings.Trim(destinationPath, EMPTY_STR) == EMPTY_STR {
// return false, errors.New("Source or Destination url is empty")
// }
// var err error
// client := grab.NewClient()
// req, _ := grab.NewRequest(destinationPath, sourcePath)
// if jwtToken != "NA" {
// req.HTTPRequest.Header.Add("Authorization", jwtToken)
// }
// var resp *grab.Response
// for index := 1; index <= retryCnt; index++ {
// resp = client.Do(req)
// //resp, err = grab.Get(destinationPath, sourcePath)
// if resp.HTTPResponse != nil && resp.HTTPResponse.StatusCode == 200 {
// return true, nil
// }
// if err != nil {
// logginghelper.LogError(" Error downloading \n", sourcePath, err)
// }
// }
// return false, errors.New("File not found on cloud")
// }
// //DownloadFile method
// func DownloadFile(downloadURL string, downloadLocation string, noOfThreads int) bool {
// url, resHeader := GetFinalurl(downloadURL)
// var downloadStatus bool
// if AcceptRanges(resHeader) {
// NoOfFiles = noOfThreads
// DownloadLocation = downloadLocation
// downloadStatus = Download(url, GetContentLength(resHeader))
// return downloadStatus
// }
// downloadStatus = DownloadSingle(url)
// return downloadStatus
// }
// //ResumeFileDownload method
// func ResumeFileDownload(downloadURL string, downloadLocation string) bool {
// DownloadLocation = downloadLocation
// BackupFile Backup File
func BackupFile(filePath string) error {
logginghelper.LogDebug("IN : BackupFile")
fileExists := filehelper.FileAvailabilityCheck(filePath)
if fileExists {
logginghelper.LogDebug("BackupFile : File Exists. Creating Backup")
parentDirectory, fileName := path.Split(filePath)
ext := path.Ext(fileName)
backupFileName := strings.Replace(fileName, ext, "", -1) + "_backup" + ext
backupFilePath := filepath.Join(parentDirectory, backupFileName)
logginghelper.LogDebug("OUT : BackupFile")
return filehelper.RenameFile(filePath, backupFilePath)
}
logginghelper.LogDebug("BackupFile : File Not Exists.")
logginghelper.LogDebug("OUT : BackupFile")
return nil
}
// url, resHeader := GetFinalurl(downloadURL)
// var resumeStatus bool
// if AcceptRanges(resHeader) {
// resumeStatus = Resume(url, GetContentLength(resHeader))
// }
// RestoreBackup Restores Backup
func RestoreBackup(filePath string) error {
logginghelper.LogDebug("IN : RestoreBackup")
fileExists := filehelper.FileAvailabilityCheck(filePath)
if fileExists {
logginghelper.LogDebug("RestoreBackup : File Exists. Deleting File")
err := filehelper.DeleteFile(filePath)
if err != nil {
logginghelper.LogError("RestoreBackup File Delete Error: ", err)
return err
}
}
parentDirectory, fileName := path.Split(filePath)
ext := path.Ext(fileName)
backupFileName := strings.Replace(fileName, ext, "", -1) + "_backup" + ext
backupFilePath := filepath.Join(parentDirectory, backupFileName)
fileExists = filehelper.FileAvailabilityCheck(backupFilePath)
if fileExists {
logginghelper.LogDebug("RestoreBackup : File Exists. Restoring Backup")
logginghelper.LogDebug("OUT : RestoreBackup")
return filehelper.RenameFile(backupFilePath, filePath)
}
logginghelper.LogDebug("RestoreBackup : Backup File Not Exists.")
logginghelper.LogDebug("OUT : RestoreBackup")
return nil
}
// return resumeStatus
// }
// DeleteBackup Deletes Backup
func DeleteBackup(filePath string) error {
logginghelper.LogDebug("IN : DeleteBackup")
parentDirectory, fileName := path.Split(filePath)
ext := path.Ext(fileName)
backupFileName := strings.Replace(fileName, ext, "", -1) + "_backup" + ext
backupFilePath := filepath.Join(parentDirectory, backupFileName)
fileExists := filehelper.FileAvailabilityCheck(backupFilePath)
if fileExists {
logginghelper.LogDebug("DeleteBackup : File Exists. Deleting Backup")
logginghelper.LogDebug("OUT : DeleteBackup")
return filehelper.DeleteFile(backupFilePath)
}
logginghelper.LogDebug("DeleteBackup : Backup File Not Exists.")
logginghelper.LogDebug("OUT : DeleteBackup")
return nil
}