Commit ca89ca29 authored by Vivek's avatar Vivek
Browse files

refacored index add operations

refacored index add operations
parent 8644b4ab
Branches
Tags
2 merge requests!169Mep release 060320,!167fdb fixes
Showing with 133 additions and 350 deletions
This diff is collapsed.
......@@ -5,7 +5,6 @@ import (
"strings"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/filemdl/filepack"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/securitymdl"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
......@@ -308,18 +307,3 @@ var lazyCallBackFnSaveIndex lazywriter.SaveDataFn = func(indexID string, data *l
return
}
}
// GenRowID generates fash for the given filename. The length of hash is 16
func GenRowID(name string) (string, error) {
name = strings.ReplaceAll(filepath.Clean(name), string(filepath.Separator), "")
rowID, err := securitymdl.GetHash(name)
if err != nil {
return "", err
}
if len(rowID) > 16 {
return rowID[:16], nil
}
return rowID, nil
}
......@@ -4,7 +4,12 @@ import (
"path/filepath"
"strings"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/errormdl"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/filemdl"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/loggermdl"
"github.com/tidwall/buntdb"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
)
// Index - Index
......@@ -29,14 +34,11 @@ func NewIndex(indexID, indexNameQuery string, IsDynamicName bool) (Index, error)
IndexNameQuery: indexNameQuery,
IsDynamicName: IsDynamicName,
}
db, err := buntdb.Open(":memory:")
var err error
idx.indexStore, err = NewStore()
if err != nil {
return idx, err
}
idx.indexStore = indexStore{
store: db,
}
return idx, nil
}
......@@ -94,9 +96,16 @@ func (i *Index) GetEntryByPath(path string) (string, error) {
return i.indexStore.Get(path)
}
func (i *Index) AddEntry(path, value string) error {
func (i *Index) AddEntry(path string, rs *gjson.Result) error {
json := `{}`
for _, indexField := range i.IndexFields {
if rs.Get(indexField.Query).Value() == nil {
return errormdl.Wrap("please provide value for index field: " + indexField.Query)
}
json, _ = sjson.Set(json, indexField.FieldName, rs.Get(indexField.Query).Value())
}
path = strings.Trim(path, string(filepath.Separator))
return i.indexStore.Set(path, value)
return i.indexStore.Set(path, json)
}
func (i *Index) AddEntries(keyValMap map[string]string) error {
......@@ -120,3 +129,71 @@ func (i *Index) DeleteMany(paths []string) error {
func (i *Index) CloseStore() error {
return i.indexStore.Close()
}
// LoadFDBIndexFromFile -
func LoadFDBIndexFromFile(indexFilePath string, fdb *FDB, indexID string) error {
loggermdl.LogError("LoadFDBIndexFromFile", indexFilePath)
index, found := fdb.GetFDBIndex(indexID)
if !found {
return errormdl.Wrap("index not found")
}
if !filemdl.FileAvailabilityCheck(indexFilePath) {
return nil
}
fileData, err := filemdl.FastReadFile(indexFilePath)
if err != nil {
loggermdl.LogError("failed to load FDB index from: ", indexFilePath)
return err
}
if len(fileData) == 0 {
return nil
}
_, fileName := filepath.Split(indexFilePath)
fileData, err = decryptData(fileData, fileName)
if err != nil {
loggermdl.LogError("failed to decrypt FDB index data: ", err)
return errormdl.Wrap("failed to decrypt FDB index data: " + err.Error())
}
data := string(fileData)
indexRecords := strings.Split(data, lineBreak)
indexDataMap := make(map[string]string)
for _, indexRecord := range indexRecords {
indexValues := strings.Split(indexRecord, IndexKeyValSeperator)
if len(indexValues) == 2 {
indexDataMap[indexValues[0]] = indexValues[1]
}
}
var fns []func(a, b string) bool
for _, idx := range index.IndexFields {
fns = append(fns, buntdb.IndexJSON(idx.FieldName))
}
loggermdl.LogError("indexDataMap", indexDataMap)
// update index file by reading all data and updating index file
return index.AddEntries(indexDataMap)
}
// LogFDBIndexFile -LogFDBIndexFile
func LogFDBIndexFile(indexFilePath string, index *Index) error {
// dbPath := filepath.Join(fdbPath, INDEXFOLDER)
// loggermdl.LogDebug("in log fdb index")
dataToStore := ``
indeKeyValMap, err := index.GetAllEntries()
if err != nil {
loggermdl.LogError(err)
return err
}
for key, value := range indeKeyValMap {
dataToStore = dataToStore + key + IndexKeyValSeperator + value + lineBreak
}
_, fileName := filepath.Split(indexFilePath)
var dataByteToWriteRes = []byte{}
var hashError error
if len(indeKeyValMap) > 0 {
dataByteToWriteRes, hashError = encryptData([]byte(dataToStore), fileName)
if errormdl.CheckErr1(hashError) != nil {
return errormdl.CheckErr1(hashError)
}
}
// dataByteToWriteRes := []byte(dataToStore)
return filemdl.WriteFile(indexFilePath, dataByteToWriteRes, true, false)
}
......@@ -5,6 +5,7 @@ import (
"strings"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/loggermdl"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/securitymdl"
"github.com/tidwall/buntdb"
"github.com/tidwall/gjson"
......@@ -137,3 +138,18 @@ func (i *indexStore) Delete(key string) error {
})
return err
}
// GenRowID generates hash for the given filename. The length of hash is 16
func GenRowID(name string) (string, error) {
name = strings.ReplaceAll(filepath.Clean(name), string(filepath.Separator), "")
rowID, err := securitymdl.GetHash(name)
if err != nil {
return "", err
}
if len(rowID) > 16 {
return rowID[:16], nil
}
return rowID, nil
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment