From 585963648883abce39c82e45a484618a4305ba07 Mon Sep 17 00:00:00 2001 From: meerkat Date: Sat, 30 Oct 2021 23:01:05 +1100 Subject: [PATCH] Go working copy Requires some fine tuning --- .gitignore | 4 +- source/golang/client/src/config/batch.no | 1 + .../golang/client/src/config/description.txt | 5 + source/golang/client/src/config/martilq.ini | 39 +++ source/golang/client/src/go.mod | 7 + source/golang/client/src/go.sum | 10 + source/golang/client/src/main.go | 189 ++++++++++++ .../golang/client/src/martilq/attributes.go | 178 +++++++++++ .../client/src/martilq/attributes_test.go | 58 ++++ source/golang/client/src/martilq/config.go | 281 ++++++++++++++++++ .../golang/client/src/martilq/config_test.go | 40 +++ source/golang/client/src/martilq/custom.go | 50 ++++ source/golang/client/src/martilq/go.mod | 9 + source/golang/client/src/martilq/go.sum | 14 + source/golang/client/src/martilq/hash.go | 42 +++ source/golang/client/src/martilq/logging.go | 10 + source/golang/client/src/martilq/marti.go | 208 +++++++++++++ .../golang/client/src/martilq/marti_test.go | 61 ++++ source/golang/client/src/martilq/resource.go | 86 ++++++ .../client/src/martilq/resource_test.go | 19 ++ source/martilq.ini | 39 +++ source/powershell/MartiLQ.ps1 | 1 + source/powershell/MartiLQItem.ps1 | 2 + source/python/client/martiLQ.py | 37 ++- 24 files changed, 1383 insertions(+), 7 deletions(-) create mode 100644 source/golang/client/src/config/batch.no create mode 100644 source/golang/client/src/config/description.txt create mode 100644 source/golang/client/src/config/martilq.ini create mode 100644 source/golang/client/src/go.mod create mode 100644 source/golang/client/src/go.sum create mode 100644 source/golang/client/src/main.go create mode 100644 source/golang/client/src/martilq/attributes.go create mode 100644 source/golang/client/src/martilq/attributes_test.go create mode 100644 source/golang/client/src/martilq/config.go create mode 100644 source/golang/client/src/martilq/config_test.go create mode 100644 source/golang/client/src/martilq/custom.go create mode 100644 source/golang/client/src/martilq/go.mod create mode 100644 source/golang/client/src/martilq/go.sum create mode 100644 source/golang/client/src/martilq/hash.go create mode 100644 source/golang/client/src/martilq/logging.go create mode 100644 source/golang/client/src/martilq/marti.go create mode 100644 source/golang/client/src/martilq/marti_test.go create mode 100644 source/golang/client/src/martilq/resource.go create mode 100644 source/golang/client/src/martilq/resource_test.go create mode 100644 source/martilq.ini diff --git a/.gitignore b/.gitignore index 9f44444..f6100fc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,6 @@ test/powershell/results/ docs/samples/powershell/test/ docs/samples/python/test -source/python/client/__pycache__ \ No newline at end of file +source/python/client/__pycache__ + +source/marti/source/golang/client/src/test diff --git a/source/golang/client/src/config/batch.no b/source/golang/client/src/config/batch.no new file mode 100644 index 0000000..95170a3 --- /dev/null +++ b/source/golang/client/src/config/batch.no @@ -0,0 +1 @@ +0005.004 \ No newline at end of file diff --git a/source/golang/client/src/config/description.txt b/source/golang/client/src/config/description.txt new file mode 100644 index 0000000..5bb63f2 --- /dev/null +++ b/source/golang/client/src/config/description.txt @@ -0,0 +1,5 @@ + +You can include more details of the MartiLQ defintion +by placing content in a text file and then +loading the contents into the JSON file before +save. diff --git a/source/golang/client/src/config/martilq.ini b/source/golang/client/src/config/martilq.ini new file mode 100644 index 0000000..59bcec0 --- /dev/null +++ b/source/golang/client/src/config/martilq.ini @@ -0,0 +1,39 @@ + +[General] + +logPath = + + +[MartiLQ] + +tags = test,sample +publisher = meerkat@merebox.com +contactPoint = Meerkat +accessLevel = Confidential +rights = None +license = MIT +batch = @./config/batch.no +theme = GOLANG + + +[Resources] + +author = +title = documentName +state = active +expires = 2:0:0 +encoding = UTF-8 +version = + +[Hash] + +hashAlgorithm = +signKey_File = +signKey_Password = + + +[Network] + +proxy = +username = +password = diff --git a/source/golang/client/src/go.mod b/source/golang/client/src/go.mod new file mode 100644 index 0000000..2e314fe --- /dev/null +++ b/source/golang/client/src/go.mod @@ -0,0 +1,7 @@ +module merebox.com/martilq_client + +go 1.16 + +replace merebox.com/martilq => ./martilq + +require merebox.com/martilq v0.0.0-00010101000000-000000000000 // indirect diff --git a/source/golang/client/src/go.sum b/source/golang/client/src/go.sum new file mode 100644 index 0000000..98c4062 --- /dev/null +++ b/source/golang/client/src/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= +gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/source/golang/client/src/main.go b/source/golang/client/src/main.go new file mode 100644 index 0000000..1daa0c8 --- /dev/null +++ b/source/golang/client/src/main.go @@ -0,0 +1,189 @@ +package main + +import ( + "fmt" + "os" + "strings" + "merebox.com/martilq" + "time" + "io/ioutil" +) + + +type Parameters struct { + task string + sourcePath string + recursive bool + configPath string + outputPath string + + title string + description string + describedBy string + landing string +} + +var params Parameters + +// go run . -- -t INIT -c ./test/my_martilq.ini +// go run . -- -t GEN -o ./test/test_martilq_directoryC.json -c ./config/martilq.ini -s ./martilq +// go run . -- -t GEN -o ./test/test_martilq_directoryC.json -c ./config/martilq.ini -s ./martilq --title "Sample run of GEN" --description "@./config/description.txt" + + +func loadArguments(args []string) { + + maxArgs := len(args) + ix := 1 + for ix < maxArgs { + matched := false + + if args[ix] == "-t" || args[ix] == "--task" { + matched = true + if ix < maxArgs { + ix = ix + 1 + params.task = strings.ToUpper(args[ix]) + } else { + panic("Missing parameter for TASK") + } + } + + if args[ix] == "-c" || args[ix] == "--config" { + matched = true + ix = ix + 1 + if ix < maxArgs { + params.configPath = args[ix] + } else { + panic("Missing parameter for CONFIG") + } + } + + if args[ix] == "-s" || args[ix] == "--source" { + matched = true + ix = ix + 1 + if ix < maxArgs { + params.sourcePath = args[ix] + } else { + panic("Missing parameter for SOURCE") + } + } + + if args[ix] == "-o" || args[ix] == "--output" { + matched = true + ix = ix + 1 + if ix < maxArgs { + params.outputPath = args[ix] + } else { + panic("Missing parameter for OUTPUT") + } + } + + + + if args[ix] == "--title" { + matched = true + if ix < maxArgs { + ix = ix + 1 + params.title = args[ix] + } else { + panic("Missing parameter for TITLE") + } + } + + if args[ix] == "--description" { + matched = true + if ix < maxArgs { + ix = ix + 1 + if args[ix][0] == '@' { + desc, err := ioutil.ReadFile(args[ix][1:]) + if err != nil { + panic("Description file not found: " + args[ix][1:]) + } + params.description = string(desc) + } else { + params.description = args[ix] + } + } else { + panic("Missing parameter for DECRIPTION") + } + } + + if args[ix] == "--landing" { + matched = true + if ix < maxArgs { + ix = ix + 1 + params.landing = args[ix] + } else { + panic("Missing parameter for LANDING") + } + } + + if !matched && args[ix] != "--" { + fmt.Println("Unrecognised command line argument: " + args[ix]) + } + + ix = ix + 1 + } + +} + +func main () { + + currentDirectory, _ := os.Getwd() + params.sourcePath = currentDirectory + //params.outputPath = "" + //params.configPath = "" + + loadArguments(os.Args) + + matched := false + + if params.task == "INIT" { + if params.configPath == "" { + panic("Missing 'config' parameter") + } + + c := martilq.NewConfiguration() + if c.SaveConfig(params.configPath) != true { + panic("Configuration not saved to: "+ params.configPath) + } + fmt.Println("Created MARTILQ INI definition: " + params.configPath) + matched = true + } + + if params.task == "GEN" { + + if params.sourcePath == "" { + panic("Missing 'source' parameter") + } + if params.outputPath == "" { + panic("Missing 'output' parameter") + } + + m := martilq.ProcessDirectory(params.configPath, params.sourcePath, params.recursive, params.outputPath ) + if params.title != "" { + m.Title = params.title + } + if params.landing != "" { + m.LandingPage = params.landing + } + if params.description != "" { + m.Description = params.description + } + m.Modified = time.Now() + m.Save(params.outputPath) + + fmt.Println("Created MARTILQ definition: " + params.outputPath) + matched = true + } + + if params.task == "RECON" { + + matched = true + } + + if matched { + fmt.Println("Completed " + params.task) + } else { + fmt.Println("Unknown task: " + params.task) + } +} diff --git a/source/golang/client/src/martilq/attributes.go b/source/golang/client/src/martilq/attributes.go new file mode 100644 index 0000000..876606b --- /dev/null +++ b/source/golang/client/src/martilq/attributes.go @@ -0,0 +1,178 @@ +package martilq + +import ( + "time" + "os" + "io" + "bytes" + "strconv" +) + +// These are predefined categories, custom ones can be added +func cCategory() [] string { + return []string {"dataset", "format", "compression", "encryption"} +} + +// These are predefined functions, custom ones can be added +// Fuctions are associated with Categories +func cFunction() [] string { + return []string {"count", "value", "temporal", "spatial", "algo"} +} + +type Attribute struct { + Category string `json:"category"` + Name string `json:"name"` + Function string `json:"function"` + Comparison string `json:"comparison"` + Value string `json:"value"` +} + + +func lineCounter(r io.Reader, lineEnding []byte) (int, error) { + buf := make([]byte, 32*1024) + count := 0 + + for { + c, err := r.Read(buf) + count += bytes.Count(buf[:c], lineEnding) + + switch { + case err == io.EOF: + return count, nil + + case err != nil: + return count, err + } + } +} + +func NewDefaultExtensionAttributes(SourcePath string, ExtendAttributes bool) []Attribute { + + var lattribute []Attribute + var oAttribute Attribute + + if ExtendAttributes { + lineEnding := []byte {'\n'} + f, err := os.Open(SourcePath) + if err != nil { + panic(err) + } + defer f.Close() + count, err := lineCounter(f, lineEnding) + + oAttribute = Attribute{"dataset","records","count","EQ", strconv.Itoa(count)} + lattribute = append(lattribute, oAttribute) + } + + return lattribute +} + + +func NewDefaultCsvAttributes(header bool, delimiter string) []Attribute { + + var lattribute []Attribute + var oAttribute Attribute + + if header { + oAttribute = Attribute{"dataset","header","count","EQ","1" } + lattribute = append(lattribute, oAttribute) + } else { + oAttribute = Attribute{"dataset","header","count","EQ","0" } + lattribute = append(lattribute, oAttribute) + } + + oAttribute = Attribute{"dataset","footer","count","EQ","0"} + lattribute = append(lattribute, oAttribute) + + oAttribute = Attribute{"format","standard","value","EQ","RFC4180"} + lattribute = append(lattribute, oAttribute) + + oAttribute = Attribute{"format","separator","value","EQ",","} + lattribute = append(lattribute, oAttribute) + + oAttribute = Attribute{"format","delimiter","value","EQ",delimiter} + lattribute = append(lattribute, oAttribute) + + oAttribute = Attribute{"format","escape","value","EQ","\""} + lattribute = append(lattribute, oAttribute) + + oAttribute = Attribute{"format","lineEnding","value","EQ","CRLF"} + lattribute = append(lattribute, oAttribute) + + oAttribute = Attribute{"format","columns","value","EQ","," } + lattribute = append(lattribute, oAttribute) + + oAttribute = Attribute{"format","radixPoint","value","EQ","." } + lattribute = append(lattribute, oAttribute) + + oAttribute = Attribute{"format","thousandSeparator","value","EQ","" } + lattribute = append(lattribute, oAttribute) + + oAttribute = Attribute{"dataset","records","count","NA","0" } + lattribute = append(lattribute, oAttribute) + + oAttribute = Attribute{"dataset","columns","count","NA","0" } + lattribute = append(lattribute, oAttribute) + + return lattribute + +} + + +func NewDefaultZipAttributes(CompressionType string, Encryption string) []Attribute { + + var attributes []Attribute + + oAttribute := Attribute {"format","compression", "algo", "EQ", CompressionType} + attributes = append(attributes, oAttribute) + + oAttribute = Attribute {"format","encryption","algo","EQ",Encryption} + attributes = append(attributes, oAttribute) + + oAttribute = Attribute {"dataset","files","count","NA", "0"} + attributes = append(attributes, oAttribute) + + return attributes +} + + +func NewDefaultTemporalAttributes(businessDate time.Time, runDate time.Time, duration bool, startDate time.Time, endDate time.Time) []Attribute { + + var attributes []Attribute + + oAttribute := Attribute {"dataset","businessDate", "temporal", "EQ", businessDate.Format("2006-01-02T15:04:05-0700")} + attributes = append(attributes, oAttribute) + + oAttribute = Attribute {"dataset","runDate", "temporal", "EQ", runDate.Format("2006-01-02T15:04:05-0700")} + attributes = append(attributes, oAttribute) + + if duration { + + oAttribute := Attribute {"dataset","duration", "temporal", "GE", startDate.Format("2006-01-02T15:04:05-0700")} + attributes = append(attributes, oAttribute) + + oAttribute = Attribute {"dataset","duration", "temporal", "LE", endDate.Format("2006-01-02T15:04:05-0700")} + attributes = append(attributes, oAttribute) + + } + + return attributes +} + + +func SetMartiAttribute(Attributes []Attribute, ACategory string, AName string, AFunction string, Comparison string , Value string) { + + for ix := 0; ix< len(Attributes); ix++ { + attr := Attributes[ix] + if attr.Category == ACategory && attr.Name == AName && attr.Function == AFunction { + Attributes[ix].Comparison = Comparison + Attributes[ix].Value = Value + return + } + } + + oAttribute := Attribute { ACategory, AName, AFunction, Comparison, Value } + Attributes = append(Attributes, oAttribute) + +} + diff --git a/source/golang/client/src/martilq/attributes_test.go b/source/golang/client/src/martilq/attributes_test.go new file mode 100644 index 0000000..a6989d6 --- /dev/null +++ b/source/golang/client/src/martilq/attributes_test.go @@ -0,0 +1,58 @@ +package martilq + +import ( + "testing" + "strconv" + "time" + "fmt" +) + +func TestAttr_Zip(t *testing.T) { + a := NewDefaultZipAttributes("7z", "") + if len(a) != 3{ + t.Error("Arrays size not 3: " + strconv.Itoa(len(a))) + } + if a[0].Value != "7z" { + t.Error("Value not saved: " + a[0].Value) + } +} + + +func TestAttr_Csv(t *testing.T) { + a := NewDefaultCsvAttributes(true,",") + if len(a) != 12 { + t.Error("Arrays size not 12: " + strconv.Itoa(len(a))) + } + if a[0].Value != "1" { + t.Error("Value not saved: " + a[0].Value) + } +} + + +func TestAttr_TemporalA(t *testing.T) { + + businessDate := time.Now() + businessDate = time.Date(businessDate.Year(), businessDate.Month(), businessDate.Day(), 0, 0, 0, 0, time.Local).AddDate(0,0,-1) + runDate := time.Now() + startDate := time.Now().AddDate(0,0,-2) + endDate := time.Now().AddDate(0,0,-1) + + a := NewDefaultTemporalAttributes(businessDate, runDate, false, startDate, endDate) + fmt.Println(a) + if len(a) != 2 { + t.Error("Arrays size not 2: " + strconv.Itoa(len(a))) + } + if a[0].Comparison != "EQ" { + t.Error("Comparison Value not saved: " + a[0].Value) + } + + a = NewDefaultTemporalAttributes(businessDate, runDate, true, startDate, endDate) + fmt.Println(a) + if len(a) != 4 { + t.Error("Arrays size not 4: " + strconv.Itoa(len(a))) + } + if a[0].Comparison != "EQ" { + t.Error("Comparison Value not saved: " + a[0].Value) + } + +} \ No newline at end of file diff --git a/source/golang/client/src/martilq/config.go b/source/golang/client/src/martilq/config.go new file mode 100644 index 0000000..e89f0fa --- /dev/null +++ b/source/golang/client/src/martilq/config.go @@ -0,0 +1,281 @@ +package martilq + +import ( + "fmt" + "gopkg.in/ini.v1" + "os" + "time" + "strings" + "strconv" +) + +const cSoftwareName = "MARTILQREFERENCE" +const cSoftwareAuthor = "Meerkat@merebox.com" +const cSoftwareVersion = "0.0.1" +const cIniFileName = "martilq.ini" +const cExpires = "7:0:0" +const cEncoding = "UTF-8" + +type configuration struct { + softwareName string + + logPath string + + publisher string + contactPoint string + accessLevel string + license string + rights string + tags string + theme string + batch string + batchInc float64 + + title string + author string + state string + version string + expires string + encoding string + + hash bool + hashAlgorithm string + signKey_File string + signKey_Password string + + proxyName string + proxyPort int + proxyUser string + proxyCredential string + + loaded bool + configPath string +} + + +func GetSoftwareName() string { + return cSoftwareName +} + +func NewConfiguration() configuration { + + c := configuration {} + + c.softwareName = GetSoftwareName() + + c.title = "documentName" + c.state = "active" + c.accessLevel = "Confidential" + c.rights = "Restricted" + c.expires = cExpires + c.encoding = cEncoding + c.batchInc = 0.001 + + c.hash = true + c.hashAlgorithm = "SHA256" + c.loaded = false + + configPath := findIni() + if configPath != "" { + c.LoadConfig(configPath) + } + + return c +} + +func findIni() string { + + foundPath := "" + + // Start wih local and move further out + if foundPath == "" { + iniFile := Loadenv("MARTILQ_MARTILQ_INI", "") + if iniFile != "" { + _, err := os.Stat(iniFile) + if err == nil { + foundPath = iniFile + } + } + } + + if foundPath == "" { + _, err := os.Stat(cIniFileName) + if err == nil { + foundPath = cIniFileName + } + } + + if foundPath == "" { + _, err := os.Stat("./config/"+ cIniFileName) + if err == nil { + foundPath = "./config/"+ cIniFileName + } + } + + if foundPath == "" { + userHomeDir, err := os.UserHomeDir() + if err == nil { + _, err := os.Stat(userHomeDir+ "/"+ cIniFileName) + if err == nil { + foundPath = userHomeDir+ "/"+ cIniFileName + } + } + } + + return foundPath +} + +func (c *configuration) SaveConfig(ConfigPath string) bool { + + cfgini, _ := ini.LooseLoad("./martilq.ini") + + cfgini.Section("General").Key("logPath").SetValue (c.logPath) + + cfgini.Section("MartiLQ").Key("tags").SetValue(c.tags) + cfgini.Section("MartiLQ").Key("publisher").SetValue(c.publisher) + cfgini.Section("MartiLQ").Key("contactPoint").SetValue(c.contactPoint) + cfgini.Section("MartiLQ").Key("accessLevel").SetValue(c.accessLevel) + cfgini.Section("MartiLQ").Key("rights").SetValue(c.rights) + cfgini.Section("MartiLQ").Key("license").SetValue(c.license) + cfgini.Section("MartiLQ").Key("batch").SetValue(c.batch) + cfgini.Section("MartiLQ").Key("theme").SetValue(c.theme) + + cfgini.Section("Resources").Key("author").SetValue (c.author) + cfgini.Section("Resources").Key("title").SetValue (c.title) + cfgini.Section("Resources").Key("state").SetValue (c.state) + cfgini.Section("Resources").Key("expires").SetValue (c.expires) + cfgini.Section("Resources").Key("encoding").SetValue (c.encoding) + cfgini.Section("Resources").Key("version").SetValue (c.version) + + cfgini.Section("Hash").Key("hashAlgorithm").SetValue (c.hashAlgorithm) + cfgini.Section("Hash").Key("signKey_File").SetValue (c.signKey_File) + cfgini.Section("Hash").Key("signKey_Password").SetValue (c.signKey_Password) + + cfgini.Section("Network").Key("proxyName").SetValue (c.proxyName) + cfgini.Section("Network").Key("proxyPort").SetValue (strconv.Itoa(c.proxyPort)) + cfgini.Section("Network").Key("proxyUser").SetValue (c.proxyUser) + cfgini.Section("Network").Key("proxyCredential").SetValue (c.proxyCredential) + + err := cfgini.SaveTo(ConfigPath) + if err != nil { + WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath)) + return false + } + return true +} + +func (c *configuration) LoadConfig(ConfigPath string) bool { + + if ConfigPath != "" { + _, err := os.Stat(ConfigPath) + if os.IsNotExist(err) { + WriteLog(fmt.Sprintf("Configuration path '%v' does not exist" , ConfigPath)) + return false + } + } else { + // Check default locations + _, err := os.Stat(cIniFileName) + if os.IsNotExist(err) == false { + ConfigPath = cIniFileName + } + + } + + if ConfigPath != "" { + + cfgini, err := ini.Load(ConfigPath) + if err != nil { + WriteLog(fmt.Sprintf("Fail to read file: %v", ConfigPath)) + fmt.Printf("Fail to read file: %v", err) + return false + } + + c.logPath = cfgini.Section("General").Key("logPath").MustString(c.logPath) + + c.tags = cfgini.Section("MartiLQ").Key("tags").MustString(c.tags) + c.accessLevel = cfgini.Section("MartiLQ").Key("accessLevel").MustString(c.accessLevel) + c.rights = cfgini.Section("MartiLQ").Key("rights").MustString(c.rights) + c.batch = cfgini.Section("MartiLQ").Key("batch").MustString(c.batch) + c.license = cfgini.Section("MartiLQ").Key("license").MustString(c.license) + c.publisher = cfgini.Section("MartiLQ").Key("publisher").MustString(c.publisher) + c.contactPoint = cfgini.Section("MartiLQ").Key("contactPoint").MustString(c.contactPoint) + c.theme= cfgini.Section("MartiLQ").Key("theme").MustString(c.theme) + + c.author = cfgini.Section("Resources").Key("title").MustString(c.title) + c.author = cfgini.Section("Resources").Key("author").MustString(c.author) + c.state = cfgini.Section("Resources").Key("state").MustString(c.state) + c.expires = cfgini.Section("Resources").Key("expires").MustString(c.expires) + c.encoding = cfgini.Section("Resources").Key("encoding").MustString(c.encoding) + + c.hashAlgorithm = cfgini.Section("Hash").Key("hashAlgorithm").MustString(c.hashAlgorithm) + c.signKey_File = cfgini.Section("Hash").Key("signKey_File").MustString(c.signKey_File) + c.signKey_Password = cfgini.Section("Hash").Key("signKey_Password").MustString(c.signKey_Password) + + c.proxyName = cfgini.Section("Network").Key("proxyName").MustString(c.proxyName) + port := cfgini.Section("Network").Key("proxyPort").MustString("") + if port != "" { + c.proxyPort, _ = strconv.Atoi(port) + } + c.proxyUser = cfgini.Section("Network").Key("proxyUser").MustString(c.proxyUser) + c.proxyCredential = cfgini.Section("Network").Key("proxyCredential").MustString(c.proxyCredential) + + WriteLog(fmt.Sprintf("Loaded config file: %v", ConfigPath)) + c.configPath = ConfigPath + } + + // Now check environmental values + c.signKey_File = Loadenv("MARTILQ_SIGNKEY_FILE", c.signKey_File) + c.signKey_Password = Loadenv("MARTILQ_SIGNKEY_PASSWORD", c.signKey_Password) + + c.logPath = Loadenv("MARTILQ_LOGPATH", c.logPath) + + c.loaded = true + + return true +} + + +func Loadenv(key string, default_value string ) string { + + tmp := os.Getenv(key) + if tmp != "" { + return tmp + } + return default_value +} + + +func (c *configuration) ExpireDate( ) time.Time { + + var expires time.Time + + lExpires := strings.Split(c.expires,":") + if len(lExpires) != 3 && len(lExpires) != 6 { + panic("Expires value '"+ c.expires +"' is invalid") + } + + + var lExpire [3]int + lex, _ := strconv.Atoi(lExpires[0]) + lExpire[0] = lex + lex, _ =strconv.Atoi(lExpires[1]) + lExpire[1] = lex + lex, _ =strconv.Atoi(lExpires[2]) + lExpire[2] = lex + + if len(lExpires) > 3 { + var lExpireD [3]int + lex, _ := strconv.Atoi(lExpires[3]) + lExpireD[0] = lex + lex, _ =strconv.Atoi(lExpires[4]) + lExpireD[1] = lex + lex, _ =strconv.Atoi(lExpires[5]) + lExpireD[2] = lex + expires = time.Now().AddDate(lExpire[0],lExpire[1],lExpire[2]).Add(time.Hour * time.Duration(lExpireD[0])).Add(time.Minute * time.Duration(lExpireD[1])).Add(time.Second * time.Duration(lExpireD[2])) + } else { + expires = time.Now().AddDate(lExpire[0],lExpire[1],lExpire[2]) + expires = time.Date(expires.Year(), expires.Month(), expires.Day(), 0, 0, 0, 0, time.Local) + } + + return expires +} diff --git a/source/golang/client/src/martilq/config_test.go b/source/golang/client/src/martilq/config_test.go new file mode 100644 index 0000000..74d8cf2 --- /dev/null +++ b/source/golang/client/src/martilq/config_test.go @@ -0,0 +1,40 @@ +package martilq + +import ( + "testing" +) + +func TestConfig_NotExist(t *testing.T) { + c := NewConfiguration() + if c.LoadConfig("../martilq.ini") != false { + t.Error("Configuration file not loaded") + } +} + +func TestConfig_LoadNone(t *testing.T) { + c := NewConfiguration() + if c.LoadConfig("") != true { + t.Error("Default configuration not loaded") + } +} + +func TestConfig_LoadFile(t *testing.T) { + c := NewConfiguration() + if c.LoadConfig("../../../../martilq.ini") != true { + t.Error("File configuration not loaded") + } + if c.state != "active" { + t.Error("State not as expected: "+c.state) + } + if c.rights != "None" { + t.Error("Rights not as expected: "+c.rights) + } +} + + +func TestConfig_Save(t *testing.T) { + c := NewConfiguration() + if c.SaveConfig("../test/martilq_write.ini") != true { + t.Error("Default configuration not saved") + } +} diff --git a/source/golang/client/src/martilq/custom.go b/source/golang/client/src/martilq/custom.go new file mode 100644 index 0000000..bda24e1 --- /dev/null +++ b/source/golang/client/src/martilq/custom.go @@ -0,0 +1,50 @@ +package martilq + +import ( + "time" +) + +type oSoftware struct { + Extension string `json:"extension"` + SoftwareName string `json:"softwareName"` + Author string `json:"author"` + Version string `json:"version"` +} + +type oTemporal struct { + Extension string `json:"extension"` + BusinessDate time.Time + RunDate time.Time +} + +type oSpatial struct { + Extension string `json:"extension"` + Country string `json:"country"` + Region string + Town string +} + +func GetSoftware() oSoftware { + + o := oSoftware {} + o.Extension = "software" + o.SoftwareName = "MARTILQREFERENCE" + o.Author = "Meerkat@merebox.com" + o.Version = "0.0.1" + + return o +} + +func GetTemporal() oTemporal { + o := oTemporal {} + o.Extension = "temporal" + + return o +} + +func GetSpatial() oSpatial { + o := oSpatial {} + o.Extension = "spatial" + + return o +} diff --git a/source/golang/client/src/martilq/go.mod b/source/golang/client/src/martilq/go.mod new file mode 100644 index 0000000..2a060d9 --- /dev/null +++ b/source/golang/client/src/martilq/go.mod @@ -0,0 +1,9 @@ +module martilq + +go 1.16 + +require ( + github.com/google/uuid v1.3.0 + github.com/stretchr/testify v1.7.0 // indirect + gopkg.in/ini.v1 v1.63.2 +) diff --git a/source/golang/client/src/martilq/go.sum b/source/golang/client/src/martilq/go.sum new file mode 100644 index 0000000..16e4088 --- /dev/null +++ b/source/golang/client/src/martilq/go.sum @@ -0,0 +1,14 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= +gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/source/golang/client/src/martilq/hash.go b/source/golang/client/src/martilq/hash.go new file mode 100644 index 0000000..15145e8 --- /dev/null +++ b/source/golang/client/src/martilq/hash.go @@ -0,0 +1,42 @@ +package martilq + +import ( + "io/ioutil" + "crypto/sha256" + "encoding/hex" + "log" +) + +type hash struct { + Algo string `json:"algo"` + Value string `json:"value"` + Signed bool `json:"signed"` +} + +func NewMartiLQHash(algo string, filePath string, value string, sign string) hash { + + h := hash {} + + h.Algo = algo + h.Value = value + h.Signed = false + + if value == "" { + hasher := sha256.New() + s, err := ioutil.ReadFile(filePath) + hasher.Write(s) + if err != nil { + log.Fatal(err) + } + + h.Value = hex.EncodeToString(hasher.Sum(nil)) + } + + if sign != "" { + + + h.Signed = true + } + + return h +} diff --git a/source/golang/client/src/martilq/logging.go b/source/golang/client/src/martilq/logging.go new file mode 100644 index 0000000..e4d24e4 --- /dev/null +++ b/source/golang/client/src/martilq/logging.go @@ -0,0 +1,10 @@ +package martilq + + +import ( + "log" +) + +func WriteLog(entry string) { + log.Printf(entry) +} diff --git a/source/golang/client/src/martilq/marti.go b/source/golang/client/src/martilq/marti.go new file mode 100644 index 0000000..c8d5a2a --- /dev/null +++ b/source/golang/client/src/martilq/marti.go @@ -0,0 +1,208 @@ +package martilq + + +import ( + "fmt" + "os" + "path/filepath" + "encoding/json" + "io/ioutil" + "github.com/google/uuid" + "math" + "time" + "errors" + "strings" + "strconv" + "bufio" + "log" + "reflect" +) + + + +type Marti struct { + Content_type string `json:"content-type"` + Title string `json:"title"` + Uid string `json:"uid"` + + Description string `json:"description"` + Modified time.Time `json:"modified"` + Publisher string `json:"publisher"` + ContactPoint string `json:"contactPoint"` + AccessLevel string `json:"accessLevel"` + Rights string `json:"rights"` + Tags []string `json:"tags"` + License string `json:"license"` + State string `json:"state"` + Batch float64 `json:"batch"` + DescribedBy string `json:"describedBy"` + LandingPage string `json:"landingPage"` + Theme string `json:"theme"` + + Resources []Resource `json:"resources"` + Custom []interface{} `json:"custom"` + + + config configuration +} + +func NewMarti() Marti { + + m := Marti {} + m.Content_type = "application/vnd.martilq.json" + u := uuid.New() + m.Uid = u.String() + + software := GetSoftware() + m.Custom = append(m.Custom, software) + spatial := GetSpatial() + m.Custom = append(m.Custom, spatial) + temporal := GetTemporal() + m.Custom = append(m.Custom, temporal) + + m.config = NewConfiguration() + + return m +} + +func Load(c configuration, pathFile string) (Marti, error) { + + m := Marti {} + + data, err := ioutil.ReadFile(pathFile) + if err != nil { + return m, err + } + + err = json.Unmarshal(data, &m) + if err != nil { + fmt.Println("error:", err) + return m, err + } + + if reflect.TypeOf(c) == reflect.TypeOf(m.config) { + m.config = c + } + + return m, nil +} + +func (m *Marti) LoadConfig(ConfigPath string) { + m.config.LoadConfig(ConfigPath) + + m.Publisher = m.config.publisher + m.ContactPoint = m.config.contactPoint + m.AccessLevel = m.config.accessLevel + m.State = m.config.state + m.Rights = m.config.rights + if m.config.tags != "" { + m.Tags = strings.Split(m.config.tags, ",") + } + m.Theme = m.config.theme + m.License = m.config.license + if m.config.batch != "" { + if m.config.batch[0] == '@' { + _, err := os.Stat(m.config.batch[1:]) + if os.IsNotExist(err) { + WriteLog(fmt.Sprintf("Batch file '%v' does not exist" , m.config.batch)) + } else { + readFile, err := os.Open(m.config.batch[1:]) + if err != nil { + log.Fatalf("failed to open file: %s", err) + } + reader := bufio.NewReader(readFile) + var line string + line, _ = reader.ReadString('\n') + m.Batch, _ = strconv.ParseFloat(line, 10) + readFile.Close() + } + + } else { + m.Batch, _ = strconv.ParseFloat(m.config.batch, 10) + } + } + +} + + +func (m *Marti) AddResource(Title string, SourcePath string, Url string) (Resource, error) { + + r, err := NewMartiLQResource(m.config, SourcePath, Url, false, true) + if err != nil { + return r, errors.New("Error in adding resource: "+SourcePath) + } + r.Title = Title + + // Find if we already have the resource + // This can occur if we are reloading + matched := false + for ix := 0; ix < len(m.Resources); ix++ { + if m.Resources[ix].DocumentName == r.DocumentName && m.Resources[ix].Url == r.Url { + m.Resources[ix] = r + matched = true + break + } + } + + if !matched { + m.Resources = append(m.Resources, r) + } + + return r, nil +} + +func (m *Marti) Save(pathFile string) bool { + + if pathFile == "" { + return false + } + + j, err := json.MarshalIndent(m, ""," ") + if err != nil { + fmt.Println(err) + return false + } else { + _ = ioutil.WriteFile(pathFile, j, 0644) + } + + return true +} + + +func ProcessDirectory(ConfigPath string, SourcePath string, Recursive bool, TargetPath string) Marti { + + m := NewMarti() + + _, err := os.Stat(TargetPath) + if err == nil { + m, err = Load(m.config, TargetPath) + if err != nil { + panic("Unable to load existing MartiLQ defintion: " + TargetPath) + } + // Update the batch number, minor version + m.Batch = math.Round((m.Batch + m.config.batchInc)/m.config.batchInc)*m.config.batchInc + m.config.LoadConfig(ConfigPath) + } else { + if ConfigPath != "" { + m.LoadConfig(ConfigPath) + } + } + + filepath.Walk(SourcePath, func(path string, info os.FileInfo, err error) error { + if err != nil { + log.Fatalf(err.Error()) + } + if info.IsDir() { + if Recursive { + + } + } else { + url := "file://"+info.Name() + m.AddResource(info.Name(), path, url) + } + return nil + }) + + return m + +} diff --git a/source/golang/client/src/martilq/marti_test.go b/source/golang/client/src/martilq/marti_test.go new file mode 100644 index 0000000..8e8bb68 --- /dev/null +++ b/source/golang/client/src/martilq/marti_test.go @@ -0,0 +1,61 @@ +package martilq + +import ( + "testing" + "os" +) + +func TestMarti_JsonSave(t *testing.T) { + m:= NewMarti() + m.Save("../test/basic_test.json") +} + +func TestMarti_ResourceAdd(t *testing.T) { + m := NewMarti() + r := NewResource(m.config) + r.Title = "Title text" + r.DocumentName = "document name" + m.Resources = append(m.Resources, r) + + r,_ = NewMartiLQResource(m.config, "marti_test.go", "https://github.com/merebox/marti", false, true) + r.Title = "Adding real file" + m.Resources = append(m.Resources, r) + + m.Save("../test/test_addresource.json") +} + +func TestMarti_ResourceExpire(t *testing.T) { + m := NewMarti() + m.LoadConfig("../../../../martilq.ini") + r := NewResource(m.config) + r.Title = "Title text" + r.DocumentName = "document name" + m.Resources = append(m.Resources, r) + + r,_ = NewMartiLQResource(m.config, "marti_test.go", "https://github.com/merebox/marti", false, true) + r.Title = "Adding real file" + m.Resources = append(m.Resources, r) + + m.Save("../test/test_addexpiry.json") +} + + +func TestMarti_DirectoryA(t *testing.T) { + + currentDirectory, _ := os.Getwd() + SourcePath := currentDirectory + Recursive := false + TargetPath := "../test/test_martilq_directoryA.json" + ProcessDirectory("", SourcePath, Recursive, TargetPath) + +} + +func TestMarti_DirectoryB(t *testing.T) { + + currentDirectory, _ := os.Getwd() + SourcePath := currentDirectory + Recursive := false + TargetPath := "../test/test_martilq_directoryB.json" + ProcessDirectory("../config/martilq.ini", SourcePath, Recursive, TargetPath) + +} diff --git a/source/golang/client/src/martilq/resource.go b/source/golang/client/src/martilq/resource.go new file mode 100644 index 0000000..e59bc7d --- /dev/null +++ b/source/golang/client/src/martilq/resource.go @@ -0,0 +1,86 @@ +package martilq + +import ( + "github.com/google/uuid" + "os" + "time" + "log" + "errors" +) + + +type Resource struct { + Title string `json:"title"` + Uid string `'json:"uid"` + DocumentName string `json:"documentName` + IssueDate time.Time `json:"issueDate"` + Modified time.Time `json:"modified"` + Expires time.Time `json:"expires"` + State string `json:"state"` + Author string `json:"author"` + Length int64 `json:"length"` + Hash hash + + Description string `json:"description"` + Url string `json:"url"` + Version string `json:"version"` + Content_Type string `json:"content-type"` + Encoding string `json:"encoding"` + Compression string `json:"compression"` + Encryption string `json:"encryption"` + + Attributes []Attribute `json:"attributes"` +} + +func NewResource(config configuration) Resource { + + r := Resource {} + u := uuid.New() + r.Uid = u.String() + + r.IssueDate = time.Now() + r.State = config.state + r.Author = config.author + r.Expires = config.ExpireDate() + r.Encoding = config.encoding + + return r +} + +func NewMartiLQResource(config configuration, sourcePath string, urlPath string, excludeHash bool, extendAttributes bool) (Resource, error) { + + r := Resource {} + + stats, err := os.Stat(sourcePath) + if err != nil { + if os.IsNotExist(err) { + log.Printf("'" + sourcePath + "' file does not exist.") + return r, errors.New("'" + sourcePath + "' file does not exist.") + } + } + + u := uuid.New() + r.Uid = u.String() + + r.State = config.state + r.Author = config.author + r.Expires = config.ExpireDate() + r.Encoding = config.encoding + + r.DocumentName = stats.Name() + if config.title == "documentName" { + r.Title = r.DocumentName + } + r.IssueDate = time.Now() + r.Modified = stats.ModTime() + r.Url = urlPath + r.Length = stats.Size() + if !excludeHash { + h := NewMartiLQHash(config.hashAlgorithm, sourcePath, "", config.signKey_File) + r.Hash = h + } + + r.Attributes = NewDefaultExtensionAttributes(sourcePath, extendAttributes) + + return r, nil +} diff --git a/source/golang/client/src/martilq/resource_test.go b/source/golang/client/src/martilq/resource_test.go new file mode 100644 index 0000000..aa75563 --- /dev/null +++ b/source/golang/client/src/martilq/resource_test.go @@ -0,0 +1,19 @@ +package martilq + +import ( + "testing" + "strconv" +) + +func TestResource_Default(t *testing.T) { + c := NewConfiguration() + r, err := NewMartiLQResource(c, "./resource.go", "", false, true) + + if err != nil { + t.Error("Error returned") + } + + if len(r.Attributes) != 1 { + t.Error("Arrays size not 1: " + strconv.Itoa(len(r.Attributes))) + } +} \ No newline at end of file diff --git a/source/martilq.ini b/source/martilq.ini new file mode 100644 index 0000000..9c96b41 --- /dev/null +++ b/source/martilq.ini @@ -0,0 +1,39 @@ + +[General] + +logPath = + + +[MartiLQ] + +tags = +publisher = +contactPoint = +accessLevel = Confidential +rights = None +license = +batch = +theme = + + +[Resources] + +author = +title = documentName +state = active +expires = 2:0:0 +encoding = UTF-8 +version = + +[Hash] + +hashAlgorithm = +signKey_File = +signKey_Password = + + +[Network] + +proxy = +username = +password = diff --git a/source/powershell/MartiLQ.ps1 b/source/powershell/MartiLQ.ps1 index 34792d0..0b437d1 100644 --- a/source/powershell/MartiLQ.ps1 +++ b/source/powershell/MartiLQ.ps1 @@ -81,6 +81,7 @@ function New-MartiDefinition [System.Collections.ArrayList]$lresource = @() $oMarti = [PSCustomObject]@{ + "content-type" = "application/vnd.martilq.json" title = "" uid = (New-Guid).ToString() resources = $lresource diff --git a/source/powershell/MartiLQItem.ps1 b/source/powershell/MartiLQItem.ps1 index e55b8be..7adad59 100644 --- a/source/powershell/MartiLQItem.ps1 +++ b/source/powershell/MartiLQItem.ps1 @@ -34,6 +34,7 @@ Param( } $lattribute = Set-MartiResourceAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes + $expires = (Get-Date).AddYears(7) $oResource = [PSCustomObject]@{ title = $item.Name.Replace($item.Extension, "") @@ -41,6 +42,7 @@ Param( documentName = $item.Name issuedDate = Get-Date -f "yyyy-MM-ddTHH:mm:ss" modified = $item.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss") + expires = $expires -f "yyyy-MM-ddTHH:mm:ss" state = "active" author = "" length = $item.Length diff --git a/source/python/client/martiLQ.py b/source/python/client/martiLQ.py index b531aa0..b3e8fd2 100644 --- a/source/python/client/martiLQ.py +++ b/source/python/client/martiLQ.py @@ -85,8 +85,16 @@ class martiLQ: self.WriteLog("Usig configuration path '{}'".format(ConfigPath)) config_object.read(ConfigPath) + if config_object.has_section("General"): + items = config_object["General"] + if not items is None: + config_attr = ["logPath"] + for x in config_attr: + if not items[x] is None and not items[x] == "": + self._oConfiguration[x] = items[x] + if config_object.has_section("Resources"): - items = config_object["Resource"] + items = config_object["Resources"] if not items is None: config_attr = ["accessLevel", "rights", "state"] for x in config_attr: @@ -255,6 +263,25 @@ class martiLQ: return self._oMartiDefinition + def Temporal(self): + + oTemporal = { + "extension": "temporal", + "businessDate": "", + "runDate": "" + } + + return oTemporal + + def Spatial(self): + + oSpatial = { + "country": "", + "region": "", + "town": "", + } + + return oSpatial def NewMartiChildItem(self, SourceFolder, UrlPath=None, Recurse=True, ExtendAttributes=True, ExcludeHash=False, Filter ="*"): @@ -327,6 +354,7 @@ class martiLQ: "documentName": item, "issuedDate": dateToday, "modified": last_modified_date, + "expires": "", "state": self.GetConfig("state"), "author": self.GetConfig("author"), "length": os.path.getsize(SourcePath), @@ -336,6 +364,7 @@ class martiLQ: "url": "", "version": "", "content-type": self.GetContentType(SourcePath), + "encoding": None, "compression": None, "encryption": None, @@ -436,18 +465,14 @@ class martiLQ: def SetMartiAttribute(self, Attributes, ACategory, AName, AFunction, Comparison, Value): - matched = False - for attr in Attributes: if attr["category"] == ACategory and attr["name"] == AName and attr["function"] == AFunction: - matched = True attr["comparison"] = Comparison attr["value"] = Value + return Attributes - if not matched: - oAttribute = { "category": ACategory, "name": AName,