Commit 297aad21 authored by Akshay Bharambe's avatar Akshay Bharambe
Browse files

Merge branch 'devbranch' into MEP_Release060220

parents 0114e7fa 34b1afbf
Branches
Tags
2 merge requests!174fdb: pack reorganization feature,!159Mep release 060220
Showing with 654 additions and 78 deletions
This diff is collapsed.
......@@ -1185,9 +1185,9 @@ func TestGetMediaFromFDB(t *testing.T) {
log.Fatal(err)
}
// dataByte = []byte("hello")
rowID := "stdId"
rowID := "4385857390816664"
// please provide record id
recordID := "1W1L8MwcroqYRyqRRKOqLJgPxRN"
recordID := "1X41JjhjwLJMRAweaH46wmKGJHR"
data, fileMeta, err := GetMediaFromFDB("myfdb", i.IndexID, rowID, recordID)
if err != nil {
log.Fatal(err)
......@@ -1441,7 +1441,7 @@ func TestExportDataFromNormalBucket(t *testing.T) {
if err != nil {
log.Fatal(err)
}
err = ExportDataAsZip("myfdb", i.IndexID, []string{}, "/home/vivekn/fdb_data/dest")
err = ExportDataAsZip("myfdb", i.IndexID, []string{}, "/home/vivekn/fdb_data/dest", MigrationTypeUpdate)
if err != nil {
log.Fatal(err)
}
......@@ -1499,7 +1499,7 @@ func TestExportDataFromPackBucket(t *testing.T) {
if err != nil {
log.Fatal(err)
}
err = ExportDataAsZip("myfdb", i.IndexID, []string{}, "/home/vivekn/fdb_data/dest")
err = ExportDataAsZip("myfdb", i.IndexID, []string{}, "/home/vivekn/fdb_data/dest", MigrationTypeUpdate)
if err != nil {
log.Fatal(err)
}
......@@ -1536,13 +1536,103 @@ func TestExportDataFromMixedBucket(t *testing.T) {
if err != nil {
log.Fatal(err)
}
err = ExportDataAsZip("myfdb", i.IndexID, []string{}, "/home/vivekn/fdb_data/dest")
err = ExportDataAsZip("myfdb", i.IndexID, []string{}, "/home/vivekn/fdb_data/dest", MigrationTypeUpdate)
if err != nil {
log.Fatal(err)
}
}
func TestExportDataBucket(t *testing.T) {
db, err := CreateFDBInstance("/home/vivekn/fdb_data/myfdb", "myfdb", false)
if err != nil {
log.Fatal("CreateFDBInstance = ", err)
}
db.EnableFDBSecurity(true)
db.EnableFDBCompression(true)
// step 1: create bucket
b := db.GetNewBucket("bucketPackMixed", false, &Bucket{})
b.SetBucketType(BucketTypeMixed)
// infileIndexProfile := filepack.InFileIndex{}
i, err := db.GetNewIndex("stdId", false)
if err != nil {
log.Fatal(err)
}
i.SetBucket(b)
fields := []IndexField{
IndexField{
FieldName: "name",
Query: "name",
},
}
i.SetFields(fields...)
// step 2: create index
err = db.CreateIndex(i)
if err != nil {
log.Fatal(err)
}
indexFilePath := filepath.Join(db.DBPath, INDEXFOLDER, i.IndexNameQuery)
err = LoadFDBIndexFromFile(indexFilePath, db, i.IndexID)
if err != nil {
log.Fatal(err)
}
zipExporter := ZipExporter{
DestPath: "/home/vivekn/fdb_data/dest",
IndexID: "stdId",
MigrationType: MigrationTypeReplace,
Queries: []string{},
FdbName: db.DBName,
}
err = zipExporter.DataExport()
if err != nil {
log.Fatal(err)
}
}
func TestImportDataIntoFDB(t *testing.T) {
err := ImportZip("myfdb", "/home/vivekn/fdb_data/dest/stdId")
db, err := CreateFDBInstance("/home/vivekn/fdb_data/importedFDB", "importedFDB", false)
if err != nil {
log.Fatal("CreateFDBInstance = ", err)
}
b := db.GetNewBucket("bucketPackMixed", false, &Bucket{})
b.SetBucketType(BucketTypeMixed)
db.EnableFDBSecurity(true)
db.EnableFDBCompression(true)
// infileIndexProfile := filepack.InFileIndex{}
i, err := db.GetNewIndex("stdId", false)
if err != nil {
log.Fatal(err)
}
i.SetBucket(b)
fields := []IndexField{
IndexField{
FieldName: "name",
Query: "name",
},
}
i.SetFields(fields...)
// step 2: create index
err = db.CreateIndex(i)
if err != nil {
log.Fatal(err)
}
zipImporter := ZipImporter{
Data: nil,
SourcePath: "/home/vivekn/fdb_data/dest/stdId",
FdbName: db.DBName,
IndexID: "stdId",
}
err = zipImporter.DataImport()
// err := ImportZip("myfdb", "/home/vivekn/fdb_data/dest/stdId")
if err != nil {
loggermdl.LogError(err)
log.Fatal(err)
}
indexFilePath := filepath.Join(db.DBPath, INDEXFOLDER, i.IndexNameQuery)
err = LogFDBIndexFile(indexFilePath, i)
if err != nil {
log.Fatal(err)
}
......
......@@ -82,6 +82,7 @@ func CreateFDBInstance(dbPath, dbName string, isDefault bool) (*FDB, error) {
buckets: make(map[string]*Bucket),
bLocker: sync.Mutex{},
restoreFileFromPack: true,
DBName: dbName,
}
if isDefault {
defaultDB = dbName
......
......@@ -77,7 +77,7 @@ type fdbSecurity struct {
userDefinedKey string // the user defined key in the project
fieldQuery string // query to get dynamic field. Ex. Each student data can be encrypted with studentID. Allowed to be empty. If empty, use same encryption for all cases.
enableSharedFiles bool
enableSharedFiles bool // True if fieldQuery is not empty.
}
var fdbSec fdbSecurity
......@@ -117,6 +117,8 @@ func GetFDBSecOptions() FDBSecOptions {
//
// Order for key: data.Get(fdbSec.fieldQuery).String() + fdbSec.encKey + fdbSec.userDefinedKey + fileName(only)
func GenSecKeyBytes(fileName string, data *gjson.Result) ([]byte, error) {
// here data is String not JSON
rs := gjson.Parse(data.String())
_, fileName = filepath.Split(fileName)
if fileName == EmptySTR {
return nil, errors.New("GenerateSecurityKey: fileName must not be empty")
......@@ -124,15 +126,14 @@ func GenSecKeyBytes(fileName string, data *gjson.Result) ([]byte, error) {
// Warning: The order of string concatenation must be preserved as specified.
key := fdbSec.encKey + fdbSec.userDefinedKey + fileName
if !strings.HasPrefix(fileName, SharedPrefix) {
// this is not a shared file.
if fdbSec.enableSharedFiles {
dynamicKey := data.Get(fdbSec.fieldQuery).String()
if dynamicKey == EmptySTR {
return nil, errors.New("GenerateSecurityKey: got empty data for defined field query")
}
key = dynamicKey + key
}
useFieldQuery, err := CheckDataForFieldQuery(fileName, &rs)
if err != nil {
return nil, err
}
if useFieldQuery {
key = rs.Get(fdbSec.fieldQuery).String() + key
}
hash, err := hashmdl.Get128BitHash([]byte(key))
if err != nil {
......@@ -151,3 +152,25 @@ func GenSecKey(fileName string, data *gjson.Result) (string, error) {
return string(key), nil
}
// CheckDataForFieldQuery checks for result of field query in data.
//
// If no query is provided OR Not shared file, the result will be false, nil
//
// Else will check for query result. If result is empty, error will be thrown.
//
// Check only if security is enabled for the database.
//
// fileName must not be empty and data must not be nil
func CheckDataForFieldQuery(fileName string, data *gjson.Result) (bool, error) {
if strings.HasPrefix(fileName, SharedPrefix) || !fdbSec.enableSharedFiles {
// this is a shared file OR no query provided. No need to check for dynamic query result.
return false, nil
}
if data.Get(fdbSec.fieldQuery).String() == EmptySTR {
return false, errors.New("CheckDataForFieldQuery: got empty data for defined field query " + fdbSec.fieldQuery)
}
return true, nil
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment