Skip to content
GitLab
Explore
Projects
Groups
Topics
Snippets
Projects
Groups
Topics
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
MKCLOS
Core Development Platform
corepkgv2
Commits
ca89ca29
Commit
ca89ca29
authored
5 years ago
by
Vivek
Browse files
Options
Downloads
Patches
Plain Diff
refacored index add operations
refacored index add operations
parent
8644b4ab
Branches
Branches containing commit
Tags
Tags containing commit
2 merge requests
!169
Mep release 060320
,
!167
fdb fixes
Changes
4
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
dalmdl/corefdb/bucket.go
+33
-327
dalmdl/corefdb/bucket.go
dalmdl/corefdb/buntdbmdl.go
+0
-16
dalmdl/corefdb/buntdbmdl.go
dalmdl/corefdb/index.go
+84
-7
dalmdl/corefdb/index.go
dalmdl/corefdb/indexStore.go
+16
-0
dalmdl/corefdb/indexStore.go
with
133 additions
and
350 deletions
dalmdl/corefdb/bucket.go
+
33
−
327
View file @
ca89ca29
This diff is collapsed.
Click to expand it.
dalmdl/corefdb/buntdbmdl.go
+
0
−
16
View file @
ca89ca29
...
...
@@ -5,7 +5,6 @@ import (
"strings"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/filemdl/filepack"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/securitymdl"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
...
...
@@ -308,18 +307,3 @@ var lazyCallBackFnSaveIndex lazywriter.SaveDataFn = func(indexID string, data *l
return
}
}
// GenRowID generates fash for the given filename. The length of hash is 16
func
GenRowID
(
name
string
)
(
string
,
error
)
{
name
=
strings
.
ReplaceAll
(
filepath
.
Clean
(
name
),
string
(
filepath
.
Separator
),
""
)
rowID
,
err
:=
securitymdl
.
GetHash
(
name
)
if
err
!=
nil
{
return
""
,
err
}
if
len
(
rowID
)
>
16
{
return
rowID
[
:
16
],
nil
}
return
rowID
,
nil
}
This diff is collapsed.
Click to expand it.
dalmdl/corefdb/index.go
+
84
−
7
View file @
ca89ca29
...
...
@@ -4,7 +4,12 @@ import (
"path/filepath"
"strings"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/errormdl"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/filemdl"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/loggermdl"
"github.com/tidwall/buntdb"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
)
// Index - Index
...
...
@@ -29,14 +34,11 @@ func NewIndex(indexID, indexNameQuery string, IsDynamicName bool) (Index, error)
IndexNameQuery
:
indexNameQuery
,
IsDynamicName
:
IsDynamicName
,
}
db
,
err
:=
buntdb
.
Open
(
":memory:"
)
var
err
error
idx
.
indexStore
,
err
=
NewStore
()
if
err
!=
nil
{
return
idx
,
err
}
idx
.
indexStore
=
indexStore
{
store
:
db
,
}
return
idx
,
nil
}
...
...
@@ -94,9 +96,16 @@ func (i *Index) GetEntryByPath(path string) (string, error) {
return
i
.
indexStore
.
Get
(
path
)
}
func
(
i
*
Index
)
AddEntry
(
path
,
value
string
)
error
{
func
(
i
*
Index
)
AddEntry
(
path
string
,
rs
*
gjson
.
Result
)
error
{
json
:=
`{}`
for
_
,
indexField
:=
range
i
.
IndexFields
{
if
rs
.
Get
(
indexField
.
Query
)
.
Value
()
==
nil
{
return
errormdl
.
Wrap
(
"please provide value for index field: "
+
indexField
.
Query
)
}
json
,
_
=
sjson
.
Set
(
json
,
indexField
.
FieldName
,
rs
.
Get
(
indexField
.
Query
)
.
Value
())
}
path
=
strings
.
Trim
(
path
,
string
(
filepath
.
Separator
))
return
i
.
indexStore
.
Set
(
path
,
value
)
return
i
.
indexStore
.
Set
(
path
,
json
)
}
func
(
i
*
Index
)
AddEntries
(
keyValMap
map
[
string
]
string
)
error
{
...
...
@@ -120,3 +129,71 @@ func (i *Index) DeleteMany(paths []string) error {
func
(
i
*
Index
)
CloseStore
()
error
{
return
i
.
indexStore
.
Close
()
}
// LoadFDBIndexFromFile -
func
LoadFDBIndexFromFile
(
indexFilePath
string
,
fdb
*
FDB
,
indexID
string
)
error
{
loggermdl
.
LogError
(
"LoadFDBIndexFromFile"
,
indexFilePath
)
index
,
found
:=
fdb
.
GetFDBIndex
(
indexID
)
if
!
found
{
return
errormdl
.
Wrap
(
"index not found"
)
}
if
!
filemdl
.
FileAvailabilityCheck
(
indexFilePath
)
{
return
nil
}
fileData
,
err
:=
filemdl
.
FastReadFile
(
indexFilePath
)
if
err
!=
nil
{
loggermdl
.
LogError
(
"failed to load FDB index from: "
,
indexFilePath
)
return
err
}
if
len
(
fileData
)
==
0
{
return
nil
}
_
,
fileName
:=
filepath
.
Split
(
indexFilePath
)
fileData
,
err
=
decryptData
(
fileData
,
fileName
)
if
err
!=
nil
{
loggermdl
.
LogError
(
"failed to decrypt FDB index data: "
,
err
)
return
errormdl
.
Wrap
(
"failed to decrypt FDB index data: "
+
err
.
Error
())
}
data
:=
string
(
fileData
)
indexRecords
:=
strings
.
Split
(
data
,
lineBreak
)
indexDataMap
:=
make
(
map
[
string
]
string
)
for
_
,
indexRecord
:=
range
indexRecords
{
indexValues
:=
strings
.
Split
(
indexRecord
,
IndexKeyValSeperator
)
if
len
(
indexValues
)
==
2
{
indexDataMap
[
indexValues
[
0
]]
=
indexValues
[
1
]
}
}
var
fns
[]
func
(
a
,
b
string
)
bool
for
_
,
idx
:=
range
index
.
IndexFields
{
fns
=
append
(
fns
,
buntdb
.
IndexJSON
(
idx
.
FieldName
))
}
loggermdl
.
LogError
(
"indexDataMap"
,
indexDataMap
)
// update index file by reading all data and updating index file
return
index
.
AddEntries
(
indexDataMap
)
}
// LogFDBIndexFile -LogFDBIndexFile
func
LogFDBIndexFile
(
indexFilePath
string
,
index
*
Index
)
error
{
// dbPath := filepath.Join(fdbPath, INDEXFOLDER)
// loggermdl.LogDebug("in log fdb index")
dataToStore
:=
``
indeKeyValMap
,
err
:=
index
.
GetAllEntries
()
if
err
!=
nil
{
loggermdl
.
LogError
(
err
)
return
err
}
for
key
,
value
:=
range
indeKeyValMap
{
dataToStore
=
dataToStore
+
key
+
IndexKeyValSeperator
+
value
+
lineBreak
}
_
,
fileName
:=
filepath
.
Split
(
indexFilePath
)
var
dataByteToWriteRes
=
[]
byte
{}
var
hashError
error
if
len
(
indeKeyValMap
)
>
0
{
dataByteToWriteRes
,
hashError
=
encryptData
([]
byte
(
dataToStore
),
fileName
)
if
errormdl
.
CheckErr1
(
hashError
)
!=
nil
{
return
errormdl
.
CheckErr1
(
hashError
)
}
}
// dataByteToWriteRes := []byte(dataToStore)
return
filemdl
.
WriteFile
(
indexFilePath
,
dataByteToWriteRes
,
true
,
false
)
}
This diff is collapsed.
Click to expand it.
dalmdl/corefdb/indexStore.go
+
16
−
0
View file @
ca89ca29
...
...
@@ -5,6 +5,7 @@ import (
"strings"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/loggermdl"
"corelab.mkcl.org/MKCLOS/coredevelopmentplatform/corepkgv2/securitymdl"
"github.com/tidwall/buntdb"
"github.com/tidwall/gjson"
...
...
@@ -137,3 +138,18 @@ func (i *indexStore) Delete(key string) error {
})
return
err
}
// GenRowID generates hash for the given filename. The length of hash is 16
func
GenRowID
(
name
string
)
(
string
,
error
)
{
name
=
strings
.
ReplaceAll
(
filepath
.
Clean
(
name
),
string
(
filepath
.
Separator
),
""
)
rowID
,
err
:=
securitymdl
.
GetHash
(
name
)
if
err
!=
nil
{
return
""
,
err
}
if
len
(
rowID
)
>
16
{
return
rowID
[
:
16
],
nil
}
return
rowID
,
nil
}
This diff is collapsed.
Click to expand it.
Preview
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment
Menu
Explore
Projects
Groups
Topics
Snippets