Go working copy

Requires some fine tuning

.gitignore

@@ -14,4 +14,6 @@ test/powershell/results/
 docs/samples/powershell/test/
 
 docs/samples/python/test
-source/python/client/__pycache__
+source/python/client/__pycache__
+
+source/marti/source/golang/client/src/test

source/golang/client/src/config/batch.no

@@ -0,0 +1 @@
+0005.004

source/golang/client/src/config/description.txt

@@ -0,0 +1,5 @@
+
+You can include more details of the MartiLQ defintion
+by placing content in a text file and then
+loading the contents into the JSON file before
+save.

source/golang/client/src/config/martilq.ini

@@ -0,0 +1,39 @@
+
+[General]
+
+logPath = 
+
+
+[MartiLQ]
+
+tags         = test,sample
+publisher    = meerkat@merebox.com
+contactPoint = Meerkat
+accessLevel  = Confidential
+rights       = None
+license      = MIT
+batch        = @./config/batch.no
+theme        = GOLANG
+
+
+[Resources]
+
+author   = 
+title    = documentName
+state    = active
+expires  = 2:0:0
+encoding = UTF-8
+version  = 
+
+[Hash]
+
+hashAlgorithm =
+signKey_File =
+signKey_Password =
+
+
+[Network]
+
+proxy = 
+username =
+password = 

source/golang/client/src/go.mod

@@ -0,0 +1,7 @@
+module merebox.com/martilq_client
+
+go 1.16
+
+replace merebox.com/martilq => ./martilq
+
+require merebox.com/martilq v0.0.0-00010101000000-000000000000 // indirect

source/golang/client/src/go.sum

@@ -0,0 +1,10 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c=
+gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

source/golang/client/src/main.go

@@ -0,0 +1,189 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"merebox.com/martilq"
+	"time"
+	"io/ioutil"
+)
+
+
+type Parameters struct {
+	task string
+	sourcePath string
+	recursive bool
+	configPath string
+	outputPath string
+
+	title string
+	description string
+	describedBy string
+	landing string
+}
+
+var params Parameters
+
+// go run . -- -t INIT -c ./test/my_martilq.ini 
+// go run . -- -t GEN -o ./test/test_martilq_directoryC.json -c ./config/martilq.ini -s ./martilq
+// go run . -- -t GEN -o ./test/test_martilq_directoryC.json -c ./config/martilq.ini -s ./martilq --title "Sample run of GEN" --description "@./config/description.txt"
+
+
+func loadArguments(args []string) {
+
+	maxArgs := len(args)
+	ix := 1
+	for ix < maxArgs {
+		matched := false
+
+		if args[ix] == "-t" || args[ix] == "--task" {
+			matched = true
+			if ix < maxArgs {
+				ix = ix + 1
+				params.task = strings.ToUpper(args[ix])
+			} else {
+				panic("Missing parameter for TASK")
+			}
+		}
+
+		if args[ix] == "-c" ||  args[ix] == "--config" {
+			matched = true
+			ix = ix + 1
+			if ix < maxArgs {
+				params.configPath = args[ix]
+			} else {
+				panic("Missing parameter for CONFIG")
+			}
+		}
+
+		if args[ix] == "-s" ||  args[ix] == "--source" {
+			matched = true
+			ix = ix + 1
+			if ix < maxArgs {
+				params.sourcePath = args[ix]
+			} else {
+				panic("Missing parameter for SOURCE")
+			}
+		}	
+
+		if args[ix] == "-o" ||  args[ix] == "--output" {
+			matched = true
+			ix = ix + 1
+			if ix < maxArgs {
+				params.outputPath = args[ix]
+			} else {
+				panic("Missing parameter for OUTPUT")
+			}
+		}	
+
+
+
+		if args[ix] == "--title" {
+			matched = true
+			if ix < maxArgs {
+				ix = ix + 1
+				params.title = args[ix]
+			} else {
+				panic("Missing parameter for TITLE")
+			}
+		}
+		
+		if args[ix] == "--description" {
+			matched = true
+			if ix < maxArgs {
+				ix = ix + 1
+				if args[ix][0] == '@' {
+					desc, err := ioutil.ReadFile(args[ix][1:])	
+					if err != nil {
+						panic("Description file not found: " + args[ix][1:])
+					} 
+					params.description = string(desc)
+				} else {
+					params.description = args[ix]
+				}
+			} else {
+				panic("Missing parameter for DECRIPTION")
+			}
+		}
+
+		if args[ix] == "--landing" {
+			matched = true
+			if ix < maxArgs {
+				ix = ix + 1
+				params.landing = args[ix]
+			} else {
+				panic("Missing parameter for LANDING")
+			}
+		}
+
+		if !matched && args[ix] != "--" {
+			fmt.Println("Unrecognised command line argument: " + args[ix])
+		}
+
+		ix = ix + 1
+	}
+
+}
+
+func main () {
+
+	currentDirectory, _ := os.Getwd()
+	params.sourcePath = currentDirectory
+	//params.outputPath = "" 
+	//params.configPath = "" 
+
+	loadArguments(os.Args)
+
+	matched := false
+
+	if params.task == "INIT" {
+		if params.configPath == "" {
+			panic("Missing 'config' parameter")
+		}
+
+		c := martilq.NewConfiguration()
+		if c.SaveConfig(params.configPath) != true {
+			panic("Configuration not saved to: "+ params.configPath)
+		}
+		fmt.Println("Created MARTILQ INI definition: " + params.configPath)
+		matched = true
+	}
+
+	if params.task == "GEN" {
+
+		if params.sourcePath == "" {
+			panic("Missing 'source' parameter")
+		}
+		if params.outputPath == "" {
+			panic("Missing 'output' parameter")
+		}
+
+		m := martilq.ProcessDirectory(params.configPath, params.sourcePath, params.recursive, params.outputPath )
+		if params.title != "" {
+			m.Title = params.title
+		}
+		if params.landing != "" {
+			m.LandingPage = params.landing
+		}
+		if params.description != "" {
+			m.Description = params.description
+		}
+		m.Modified = time.Now()
+		m.Save(params.outputPath)
+
+		fmt.Println("Created MARTILQ definition: " + params.outputPath)
+		matched = true
+	}
+
+	if params.task == "RECON" {
+		
+		matched = true
+	}
+
+	if matched {
+		fmt.Println("Completed " + params.task)
+	} else {
+		fmt.Println("Unknown task: " + params.task)
+	}
+}

source/golang/client/src/martilq/attributes.go

@@ -0,0 +1,178 @@
+package martilq
+
+import (
+	"time"	
+	"os"
+	"io"
+	"bytes"
+	"strconv"
+)
+
+// These are predefined categories, custom ones can be added
+func cCategory() [] string {
+	return []string {"dataset", "format", "compression", "encryption"}
+}
+
+// These are predefined functions, custom ones can be added
+// Fuctions are associated with Categories
+func cFunction() [] string {
+	return []string {"count", "value", "temporal", "spatial", "algo"}
+}
+
+type Attribute struct {
+	Category string `json:"category"`
+	Name string `json:"name"`
+	Function string `json:"function"`
+	Comparison string `json:"comparison"`
+	Value string `json:"value"`
+}
+
+
+func lineCounter(r io.Reader, lineEnding []byte) (int, error) {
+    buf := make([]byte, 32*1024)
+    count := 0
+
+    for {
+        c, err := r.Read(buf)
+        count += bytes.Count(buf[:c], lineEnding)
+
+        switch {
+        case err == io.EOF:
+            return count, nil
+
+        case err != nil:
+            return count, err
+        }
+    }
+}
+
+func NewDefaultExtensionAttributes(SourcePath string, ExtendAttributes bool) []Attribute {
+        
+	var lattribute []Attribute
+	var oAttribute Attribute
+
+	if ExtendAttributes {
+		lineEnding := []byte {'\n'}
+		f, err := os.Open(SourcePath)
+		if err != nil {
+			panic(err)
+		}
+		defer f.Close()
+		count, err := lineCounter(f, lineEnding)
+
+		oAttribute = Attribute{"dataset","records","count","EQ", strconv.Itoa(count)}
+		lattribute = append(lattribute, oAttribute)
+	}
+		
+	return lattribute
+}
+
+
+func NewDefaultCsvAttributes(header bool, delimiter string) []Attribute {
+        
+	var lattribute []Attribute
+	var oAttribute Attribute
+
+	if header {
+		oAttribute = Attribute{"dataset","header","count","EQ","1"	}
+		lattribute = append(lattribute, oAttribute)
+	} else {
+		oAttribute = Attribute{"dataset","header","count","EQ","0"	}
+		lattribute = append(lattribute, oAttribute)
+	}
+
+	oAttribute = Attribute{"dataset","footer","count","EQ","0"}
+	lattribute = append(lattribute, oAttribute)
+
+	oAttribute = Attribute{"format","standard","value","EQ","RFC4180"}
+	lattribute = append(lattribute, oAttribute)
+
+	oAttribute = Attribute{"format","separator","value","EQ",","}
+	lattribute = append(lattribute, oAttribute)
+
+	oAttribute = Attribute{"format","delimiter","value","EQ",delimiter}
+	lattribute = append(lattribute, oAttribute)
+
+	oAttribute = Attribute{"format","escape","value","EQ","\""}
+	lattribute = append(lattribute, oAttribute)
+
+	oAttribute = Attribute{"format","lineEnding","value","EQ","CRLF"}
+	lattribute = append(lattribute, oAttribute)
+
+	oAttribute = Attribute{"format","columns","value","EQ",","	}
+	lattribute = append(lattribute, oAttribute)
+
+	oAttribute = Attribute{"format","radixPoint","value","EQ","."	}
+	lattribute = append(lattribute, oAttribute)
+
+	oAttribute = Attribute{"format","thousandSeparator","value","EQ",""	}
+	lattribute = append(lattribute, oAttribute)
+
+	oAttribute = Attribute{"dataset","records","count","NA","0"	}
+	lattribute = append(lattribute, oAttribute)
+
+	oAttribute = Attribute{"dataset","columns","count","NA","0"	}
+	lattribute = append(lattribute, oAttribute)
+
+	return lattribute
+
+}
+
+
+func NewDefaultZipAttributes(CompressionType string, Encryption string) []Attribute {
+        
+	var attributes []Attribute
+
+	oAttribute := Attribute {"format","compression", "algo", "EQ", CompressionType}
+	attributes = append(attributes, oAttribute)
+
+	oAttribute = Attribute {"format","encryption","algo","EQ",Encryption}
+	attributes = append(attributes, oAttribute)
+
+	oAttribute = Attribute {"dataset","files","count","NA", "0"}
+	attributes = append(attributes, oAttribute)
+		
+	return attributes
+}
+
+
+func NewDefaultTemporalAttributes(businessDate time.Time, runDate time.Time, duration bool, startDate time.Time, endDate time.Time) []Attribute {
+        
+	var attributes []Attribute
+
+	oAttribute := Attribute {"dataset","businessDate", "temporal", "EQ", businessDate.Format("2006-01-02T15:04:05-0700")}
+	attributes = append(attributes, oAttribute)
+
+	oAttribute = Attribute {"dataset","runDate", "temporal", "EQ", runDate.Format("2006-01-02T15:04:05-0700")}
+	attributes = append(attributes, oAttribute)
+
+	if duration {
+
+		oAttribute := Attribute {"dataset","duration", "temporal", "GE", startDate.Format("2006-01-02T15:04:05-0700")}
+		attributes = append(attributes, oAttribute)
+	
+		oAttribute = Attribute {"dataset","duration", "temporal", "LE", endDate.Format("2006-01-02T15:04:05-0700")}
+		attributes = append(attributes, oAttribute)
+	
+	}
+		
+	return attributes
+}
+
+
+func SetMartiAttribute(Attributes []Attribute, ACategory string, AName string, AFunction string, Comparison string , Value string) {
+
+	for ix := 0; ix< len(Attributes); ix++ {
+		attr := Attributes[ix]
+		if attr.Category == ACategory && attr.Name == AName && attr.Function == AFunction {
+			Attributes[ix].Comparison = Comparison
+			Attributes[ix].Value = Value
+			return
+		}
+	}
+
+	oAttribute := Attribute { ACategory, AName, AFunction, Comparison, Value }	
+	Attributes = append(Attributes, oAttribute)
+
+}
+

source/golang/client/src/martilq/attributes_test.go

@@ -0,0 +1,58 @@
+package martilq
+
+import (
+	"testing"
+	"strconv"
+	"time"
+	"fmt"
+)
+
+func TestAttr_Zip(t *testing.T) {
+	a := NewDefaultZipAttributes("7z", "")
+	if len(a) != 3{
+		t.Error("Arrays size not 3: " + strconv.Itoa(len(a)))
+	}
+	if a[0].Value != "7z" {
+		t.Error("Value not saved: " + a[0].Value)
+	}
+}
+
+
+func TestAttr_Csv(t *testing.T) {
+	a := NewDefaultCsvAttributes(true,",")
+	if len(a) != 12 {
+		t.Error("Arrays size not 12: " + strconv.Itoa(len(a)))
+	}
+	if a[0].Value != "1" {
+		t.Error("Value not saved: " + a[0].Value)
+	}
+}
+
+
+func TestAttr_TemporalA(t *testing.T) {
+
+	businessDate := time.Now()
+	businessDate = time.Date(businessDate.Year(), businessDate.Month(), businessDate.Day(), 0, 0, 0, 0, time.Local).AddDate(0,0,-1)
+	runDate := time.Now()
+	startDate := time.Now().AddDate(0,0,-2)
+	endDate := time.Now().AddDate(0,0,-1)
+
+	a := NewDefaultTemporalAttributes(businessDate, runDate, false, startDate, endDate)
+	fmt.Println(a)
+	if len(a) != 2 {
+		t.Error("Arrays size not 2: " + strconv.Itoa(len(a)))
+	}
+	if a[0].Comparison != "EQ" {
+		t.Error("Comparison Value not saved: " + a[0].Value)
+	}
+
+	a = NewDefaultTemporalAttributes(businessDate, runDate, true, startDate, endDate)
+	fmt.Println(a)
+	if len(a) != 4 {
+		t.Error("Arrays size not 4: " + strconv.Itoa(len(a)))
+	}
+	if a[0].Comparison != "EQ" {
+		t.Error("Comparison Value not saved: " + a[0].Value)
+	}
+
+}

source/golang/client/src/martilq/config.go

@@ -0,0 +1,281 @@
+package martilq
+
+import (
+	"fmt"
+	"gopkg.in/ini.v1"
+	"os"
+	"time"
+	"strings"
+	"strconv"
+)
+
+const cSoftwareName = "MARTILQREFERENCE"
+const cSoftwareAuthor = "Meerkat@merebox.com"
+const cSoftwareVersion = "0.0.1"
+const cIniFileName = "martilq.ini"
+const cExpires = "7:0:0"
+const cEncoding = "UTF-8"
+
+type configuration struct {
+	softwareName string
+
+	logPath string
+
+	publisher string
+	contactPoint string
+	accessLevel string
+	license string
+	rights string
+	tags string
+	theme string
+	batch string
+	batchInc float64
+
+	title string
+	author string
+	state string
+	version string
+	expires string
+	encoding string
+
+	hash bool
+	hashAlgorithm string
+	signKey_File string
+	signKey_Password string
+
+	proxyName string
+	proxyPort int
+	proxyUser string
+	proxyCredential string
+
+	loaded bool
+	configPath string
+}
+
+
+func GetSoftwareName() string {
+	return cSoftwareName
+}
+
+func NewConfiguration() configuration {
+
+	c :=  configuration {}
+
+	c.softwareName = GetSoftwareName()
+
+	c.title = "documentName"
+	c.state = "active"
+	c.accessLevel = "Confidential"
+	c.rights = "Restricted"
+	c.expires = cExpires
+	c.encoding = cEncoding
+	c.batchInc = 0.001
+
+	c.hash = true
+	c.hashAlgorithm = "SHA256" 
+	c.loaded = false
+
+	configPath := findIni()
+	if configPath != "" {
+		c.LoadConfig(configPath)
+	}
+
+	return c
+}
+
+func findIni() string {
+
+	foundPath := ""
+
+	// Start wih local and move further out
+	if foundPath == "" {
+		iniFile := Loadenv("MARTILQ_MARTILQ_INI", "")
+			if iniFile != "" {
+			_, err := os.Stat(iniFile)
+			if err == nil {
+				foundPath = iniFile
+			}
+		}
+	}
+
+	if foundPath == "" {
+		_, err := os.Stat(cIniFileName)
+		if err == nil {
+			foundPath = cIniFileName
+		}
+	}
+
+	if foundPath == "" {
+		_, err := os.Stat("./config/"+ cIniFileName)
+		if err == nil {
+			foundPath = "./config/"+  cIniFileName
+		}
+	}
+
+	if foundPath == "" {
+		userHomeDir, err := os.UserHomeDir()
+		if err == nil {
+			_, err := os.Stat(userHomeDir+ "/"+ cIniFileName)
+			if err == nil {
+				foundPath = userHomeDir+ "/"+ cIniFileName
+			}
+		}
+	}
+
+	return foundPath
+}
+
+func (c *configuration) SaveConfig(ConfigPath string) bool {
+
+	cfgini, _ := ini.LooseLoad("./martilq.ini")
+		
+	cfgini.Section("General").Key("logPath").SetValue (c.logPath)
+
+	cfgini.Section("MartiLQ").Key("tags").SetValue(c.tags)
+	cfgini.Section("MartiLQ").Key("publisher").SetValue(c.publisher)
+	cfgini.Section("MartiLQ").Key("contactPoint").SetValue(c.contactPoint)
+	cfgini.Section("MartiLQ").Key("accessLevel").SetValue(c.accessLevel)
+	cfgini.Section("MartiLQ").Key("rights").SetValue(c.rights)
+	cfgini.Section("MartiLQ").Key("license").SetValue(c.license)
+	cfgini.Section("MartiLQ").Key("batch").SetValue(c.batch)
+	cfgini.Section("MartiLQ").Key("theme").SetValue(c.theme)
+
+	cfgini.Section("Resources").Key("author").SetValue (c.author)
+	cfgini.Section("Resources").Key("title").SetValue (c.title)
+	cfgini.Section("Resources").Key("state").SetValue (c.state)
+	cfgini.Section("Resources").Key("expires").SetValue (c.expires)
+	cfgini.Section("Resources").Key("encoding").SetValue (c.encoding)
+	cfgini.Section("Resources").Key("version").SetValue (c.version)
+	
+	cfgini.Section("Hash").Key("hashAlgorithm").SetValue (c.hashAlgorithm)
+	cfgini.Section("Hash").Key("signKey_File").SetValue (c.signKey_File)
+	cfgini.Section("Hash").Key("signKey_Password").SetValue (c.signKey_Password)
+
+	cfgini.Section("Network").Key("proxyName").SetValue (c.proxyName)
+	cfgini.Section("Network").Key("proxyPort").SetValue (strconv.Itoa(c.proxyPort))
+	cfgini.Section("Network").Key("proxyUser").SetValue (c.proxyUser)
+	cfgini.Section("Network").Key("proxyCredential").SetValue (c.proxyCredential)
+
+	err := cfgini.SaveTo(ConfigPath)
+	if err != nil {
+		WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath))
+		return false
+	}
+	return true
+}
+
+func (c *configuration) LoadConfig(ConfigPath string) bool {
+
+	if ConfigPath != "" {
+		_, err := os.Stat(ConfigPath)
+		if os.IsNotExist(err) {
+			WriteLog(fmt.Sprintf("Configuration path '%v' does not exist" , ConfigPath))
+			return false
+		}
+	} else {
+		// Check default locations
+		_, err := os.Stat(cIniFileName)
+		if os.IsNotExist(err) == false {
+			ConfigPath = cIniFileName
+		}
+
+	}
+
+	if ConfigPath != "" {
+
+		cfgini, err := ini.Load(ConfigPath)
+		if err != nil {
+			WriteLog(fmt.Sprintf("Fail to read file: %v", ConfigPath))
+			fmt.Printf("Fail to read file: %v", err)
+			return false
+		}
+		
+		c.logPath = cfgini.Section("General").Key("logPath").MustString(c.logPath)
+
+		c.tags = cfgini.Section("MartiLQ").Key("tags").MustString(c.tags)
+		c.accessLevel = cfgini.Section("MartiLQ").Key("accessLevel").MustString(c.accessLevel)
+		c.rights = cfgini.Section("MartiLQ").Key("rights").MustString(c.rights)
+		c.batch = cfgini.Section("MartiLQ").Key("batch").MustString(c.batch)
+		c.license = cfgini.Section("MartiLQ").Key("license").MustString(c.license)
+		c.publisher = cfgini.Section("MartiLQ").Key("publisher").MustString(c.publisher)
+		c.contactPoint = cfgini.Section("MartiLQ").Key("contactPoint").MustString(c.contactPoint)
+		c.theme= cfgini.Section("MartiLQ").Key("theme").MustString(c.theme)
+
+		c.author = cfgini.Section("Resources").Key("title").MustString(c.title)
+		c.author = cfgini.Section("Resources").Key("author").MustString(c.author)
+		c.state = cfgini.Section("Resources").Key("state").MustString(c.state)
+		c.expires = cfgini.Section("Resources").Key("expires").MustString(c.expires)
+		c.encoding = cfgini.Section("Resources").Key("encoding").MustString(c.encoding)
+		
+		c.hashAlgorithm = cfgini.Section("Hash").Key("hashAlgorithm").MustString(c.hashAlgorithm)
+		c.signKey_File = cfgini.Section("Hash").Key("signKey_File").MustString(c.signKey_File)
+		c.signKey_Password = cfgini.Section("Hash").Key("signKey_Password").MustString(c.signKey_Password)
+
+		c.proxyName = cfgini.Section("Network").Key("proxyName").MustString(c.proxyName)
+		port := cfgini.Section("Network").Key("proxyPort").MustString("")
+		if port != "" {
+			c.proxyPort, _ = strconv.Atoi(port)
+		}
+		c.proxyUser = cfgini.Section("Network").Key("proxyUser").MustString(c.proxyUser)
+		c.proxyCredential = cfgini.Section("Network").Key("proxyCredential").MustString(c.proxyCredential)
+	
+		WriteLog(fmt.Sprintf("Loaded config file: %v", ConfigPath))
+		c.configPath = ConfigPath
+	}
+
+	// Now check environmental values
+	c.signKey_File = Loadenv("MARTILQ_SIGNKEY_FILE", c.signKey_File)
+	c.signKey_Password = Loadenv("MARTILQ_SIGNKEY_PASSWORD", c.signKey_Password)
+	
+	c.logPath = Loadenv("MARTILQ_LOGPATH", c.logPath)
+
+	c.loaded = true
+	
+	return true
+}
+
+
+func Loadenv(key string, default_value string ) string {
+
+	tmp := os.Getenv(key)
+	if tmp != "" {
+		return tmp
+	}
+	return default_value
+}
+
+
+func (c *configuration) ExpireDate( ) time.Time {
+
+	var expires time.Time
+
+	lExpires := strings.Split(c.expires,":")
+	if len(lExpires) != 3 && len(lExpires) != 6 {
+		panic("Expires value '"+ c.expires +"' is invalid")
+	}
+
+
+	var lExpire [3]int 
+	lex, _ := strconv.Atoi(lExpires[0])
+	lExpire[0] = lex
+	lex, _ =strconv.Atoi(lExpires[1])
+	lExpire[1] = lex
+	lex, _ =strconv.Atoi(lExpires[2])
+	lExpire[2] = lex
+
+	if len(lExpires) > 3 {
+		var lExpireD [3]int
+		lex, _ := strconv.Atoi(lExpires[3])
+		lExpireD[0] = lex
+		lex, _ =strconv.Atoi(lExpires[4])
+		lExpireD[1] = lex
+		lex, _ =strconv.Atoi(lExpires[5])
+		lExpireD[2] = lex
+		expires = time.Now().AddDate(lExpire[0],lExpire[1],lExpire[2]).Add(time.Hour * time.Duration(lExpireD[0])).Add(time.Minute * time.Duration(lExpireD[1])).Add(time.Second * time.Duration(lExpireD[2]))
+	} else {
+		expires = time.Now().AddDate(lExpire[0],lExpire[1],lExpire[2])
+		expires = time.Date(expires.Year(), expires.Month(), expires.Day(), 0, 0, 0, 0, time.Local)
+	}
+
+	return expires
+}

source/golang/client/src/martilq/config_test.go

@@ -0,0 +1,40 @@
+package martilq
+
+import (
+	"testing"
+)
+
+func TestConfig_NotExist(t *testing.T) {
+	c := NewConfiguration()
+	if c.LoadConfig("../martilq.ini") != false {
+		t.Error("Configuration file not loaded")
+	}
+}
+
+func TestConfig_LoadNone(t *testing.T) {
+	c := NewConfiguration()
+	if c.LoadConfig("") != true {
+		t.Error("Default configuration not loaded")
+	}
+}
+
+func TestConfig_LoadFile(t *testing.T) {
+	c := NewConfiguration()
+	if c.LoadConfig("../../../../martilq.ini") != true {
+		t.Error("File configuration not loaded")
+	}
+	if c.state != "active" {
+		t.Error("State not as expected: "+c.state)
+	}
+	if c.rights != "None" {
+		t.Error("Rights not as expected: "+c.rights)
+	}
+}
+
+
+func TestConfig_Save(t *testing.T) {
+	c := NewConfiguration()
+	if c.SaveConfig("../test/martilq_write.ini") != true {
+		t.Error("Default configuration not saved")
+	}
+}

source/golang/client/src/martilq/custom.go

@@ -0,0 +1,50 @@
+package martilq
+
+import (
+	"time"
+)
+       
+type oSoftware struct {
+	Extension string `json:"extension"` 
+	SoftwareName string `json:"softwareName"` 
+	Author string `json:"author"` 
+	Version string `json:"version"`
+}
+
+type oTemporal struct {
+	Extension string `json:"extension"`
+	BusinessDate time.Time
+	RunDate time.Time
+}
+
+type oSpatial struct {
+	Extension string `json:"extension"`
+	Country string `json:"country"`
+	Region string
+	Town string
+}
+
+func GetSoftware() oSoftware {
+
+	o := oSoftware {}
+	o.Extension = "software"
+	o.SoftwareName = "MARTILQREFERENCE"
+	o.Author = "Meerkat@merebox.com"
+	o.Version = "0.0.1"
+
+	return o
+}
+
+func GetTemporal() oTemporal {
+	o := oTemporal {}
+	o.Extension = "temporal"
+
+	return o
+}
+
+func GetSpatial() oSpatial {
+	o := oSpatial {}
+	o.Extension = "spatial"
+
+	return o
+}

source/golang/client/src/martilq/go.mod

@@ -0,0 +1,9 @@
+module martilq
+
+go 1.16
+
+require (
+	github.com/google/uuid v1.3.0
+	github.com/stretchr/testify v1.7.0 // indirect
+	gopkg.in/ini.v1 v1.63.2
+)

source/golang/client/src/martilq/go.sum

@@ -0,0 +1,14 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c=
+gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

source/golang/client/src/martilq/hash.go

@@ -0,0 +1,42 @@
+package martilq
+
+import (
+    "io/ioutil"
+	"crypto/sha256"
+	"encoding/hex"
+	"log"
+)
+
+type hash struct {
+	Algo string `json:"algo"`
+	Value string `json:"value"`
+	Signed bool `json:"signed"`
+}
+
+func NewMartiLQHash(algo string, filePath string, value string, sign string) hash {
+
+	h := hash {}
+
+	h.Algo = algo
+	h.Value = value
+	h.Signed = false
+
+	if value == "" {
+		hasher := sha256.New()
+		s, err := ioutil.ReadFile(filePath)    
+		hasher.Write(s)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		h.Value = hex.EncodeToString(hasher.Sum(nil))
+	}
+
+	if sign != "" {
+
+
+		h.Signed = true
+	}
+
+	return h
+}

source/golang/client/src/martilq/logging.go

@@ -0,0 +1,10 @@
+package martilq
+
+
+import (
+	"log"
+)
+
+func WriteLog(entry string) {
+	log.Printf(entry)
+}

source/golang/client/src/martilq/marti.go

@@ -0,0 +1,208 @@
+package martilq
+
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"encoding/json"
+	"io/ioutil"
+	"github.com/google/uuid"
+	"math"
+	"time"
+	"errors"
+	"strings"
+	"strconv"
+	"bufio"
+	"log"
+	"reflect"
+)
+
+
+
+type Marti struct {
+	Content_type string `json:"content-type"`
+	Title string `json:"title"`
+	Uid string `json:"uid"`
+
+	Description string `json:"description"`
+	Modified time.Time `json:"modified"`
+	Publisher string `json:"publisher"`
+	ContactPoint string `json:"contactPoint"`
+	AccessLevel string `json:"accessLevel"`
+	Rights string `json:"rights"`
+	Tags []string `json:"tags"`
+	License string `json:"license"`
+	State string `json:"state"`
+	Batch float64 `json:"batch"`
+	DescribedBy string `json:"describedBy"`
+	LandingPage string `json:"landingPage"`
+	Theme string `json:"theme"`
+
+	Resources []Resource `json:"resources"`
+	Custom []interface{} `json:"custom"`
+
+
+	config configuration
+}
+
+func NewMarti() Marti {
+
+	m := Marti {}
+	m.Content_type = "application/vnd.martilq.json"
+	u := uuid.New()
+	m.Uid = u.String()
+
+	software := GetSoftware()
+	m.Custom = append(m.Custom, software)
+	spatial := GetSpatial()
+	m.Custom = append(m.Custom, spatial)
+	temporal := GetTemporal()
+	m.Custom = append(m.Custom, temporal)
+
+	m.config = NewConfiguration()
+
+	return m
+}
+
+func Load(c configuration, pathFile string) (Marti, error) {
+	
+	m := Marti {}
+
+	data, err := ioutil.ReadFile(pathFile)
+    if err != nil {
+	  return m, err
+    }
+
+    err = json.Unmarshal(data, &m)
+    if err != nil {
+        fmt.Println("error:", err)
+		return m, err
+    }
+
+	if reflect.TypeOf(c) == reflect.TypeOf(m.config) {
+		m.config = c
+	}
+
+	return m, nil
+}
+
+func (m *Marti) LoadConfig(ConfigPath string) {
+	m.config.LoadConfig(ConfigPath)
+
+	m.Publisher = m.config.publisher
+	m.ContactPoint = m.config.contactPoint
+	m.AccessLevel = m.config.accessLevel
+	m.State = m.config.state
+	m.Rights = m.config.rights
+	if m.config.tags != "" {
+		m.Tags = strings.Split(m.config.tags, ",")
+	}
+	m.Theme = m.config.theme
+	m.License = m.config.license
+	if m.config.batch != "" {
+		if m.config.batch[0] == '@' {
+			_, err := os.Stat(m.config.batch[1:])
+			if os.IsNotExist(err) {
+				WriteLog(fmt.Sprintf("Batch file '%v' does not exist" , m.config.batch))		
+			} else {
+				readFile, err := os.Open(m.config.batch[1:])
+				if err != nil {
+					log.Fatalf("failed to open file: %s", err)
+				}
+				reader := bufio.NewReader(readFile)
+				var line string
+				line, _ = reader.ReadString('\n')
+				m.Batch, _ = strconv.ParseFloat(line, 10)
+				readFile.Close()
+			}
+			
+		} else {
+			m.Batch, _ = strconv.ParseFloat(m.config.batch, 10)
+		}
+	}
+
+}
+
+
+func (m *Marti) AddResource(Title string, SourcePath string, Url string) (Resource, error) {
+
+	r, err := NewMartiLQResource(m.config, SourcePath, Url, false, true)
+	if err != nil {
+		return r, errors.New("Error in adding resource: "+SourcePath)
+	}
+	r.Title = Title
+	
+	// Find if we already have the resource
+	// This can occur if we are reloading
+	matched := false
+	for ix := 0; ix < len(m.Resources); ix++ {
+		if m.Resources[ix].DocumentName == r.DocumentName && m.Resources[ix].Url == r.Url {
+			m.Resources[ix] = r
+			matched = true
+			break
+		}		
+    }
+
+	if !matched {
+		m.Resources = append(m.Resources, r)
+	}
+
+	return r, nil
+}
+
+func (m *Marti) Save(pathFile string) bool {
+
+	if pathFile == "" {
+		return false
+	}
+
+	j, err := json.MarshalIndent(m, "","    ")
+	if err != nil {
+		fmt.Println(err)
+		return false
+	} else {
+		_ = ioutil.WriteFile(pathFile, j, 0644)
+	}
+
+	return true
+}
+
+
+func ProcessDirectory(ConfigPath string, SourcePath string, Recursive bool, TargetPath string) Marti {
+
+	m := NewMarti()
+
+	_, err := os.Stat(TargetPath)
+	if err == nil {
+		m, err = Load(m.config, TargetPath)
+		if err != nil {
+			panic("Unable to load existing MartiLQ defintion: " + TargetPath)
+		}
+		// Update the batch number, minor version
+		m.Batch = math.Round((m.Batch + m.config.batchInc)/m.config.batchInc)*m.config.batchInc
+		m.config.LoadConfig(ConfigPath)
+	} else {
+		if ConfigPath != "" {
+			m.LoadConfig(ConfigPath)
+		}
+	}
+
+	filepath.Walk(SourcePath, func(path string, info os.FileInfo, err error) error {
+        if err != nil {
+            log.Fatalf(err.Error())
+        }
+		if info.IsDir() {
+			if Recursive {
+
+			}
+		} else {
+			url := "file://"+info.Name()
+			m.AddResource(info.Name(), path, url) 
+		}
+        return nil
+    })
+
+	return m
+
+}

source/golang/client/src/martilq/marti_test.go

@@ -0,0 +1,61 @@
+package martilq
+
+import (
+	"testing"
+	"os"
+)
+
+func TestMarti_JsonSave(t *testing.T) {
+	m:= NewMarti()
+	m.Save("../test/basic_test.json")
+}
+
+func TestMarti_ResourceAdd(t *testing.T) {
+	m := NewMarti()
+	r := NewResource(m.config)
+	r.Title = "Title text"
+	r.DocumentName = "document name"
+	m.Resources = append(m.Resources, r)
+
+	r,_ = NewMartiLQResource(m.config, "marti_test.go", "https://github.com/merebox/marti", false, true)
+	r.Title = "Adding real file"
+	m.Resources = append(m.Resources, r)
+
+	m.Save("../test/test_addresource.json")
+}
+
+func TestMarti_ResourceExpire(t *testing.T) {
+	m := NewMarti()
+	m.LoadConfig("../../../../martilq.ini")
+	r := NewResource(m.config)
+	r.Title = "Title text"
+	r.DocumentName = "document name"
+	m.Resources = append(m.Resources, r)
+
+	r,_ = NewMartiLQResource(m.config, "marti_test.go", "https://github.com/merebox/marti", false, true)
+	r.Title = "Adding real file"
+	m.Resources = append(m.Resources, r)
+
+	m.Save("../test/test_addexpiry.json")
+}
+
+
+func TestMarti_DirectoryA(t *testing.T) {
+	
+	currentDirectory, _ := os.Getwd()
+	SourcePath := currentDirectory
+	Recursive := false
+	TargetPath := "../test/test_martilq_directoryA.json"
+	ProcessDirectory("", SourcePath, Recursive, TargetPath) 
+
+}
+
+func TestMarti_DirectoryB(t *testing.T) {
+	
+	currentDirectory, _ := os.Getwd()
+	SourcePath := currentDirectory
+	Recursive := false
+	TargetPath := "../test/test_martilq_directoryB.json"
+	ProcessDirectory("../config/martilq.ini", SourcePath, Recursive, TargetPath) 
+
+}

source/golang/client/src/martilq/resource.go

@@ -0,0 +1,86 @@
+package martilq
+
+import (
+	"github.com/google/uuid"
+	"os"
+	"time"
+	"log"
+	"errors"
+)
+
+
+type Resource struct {
+	Title string `json:"title"`
+	Uid string `'json:"uid"`
+	DocumentName string `json:"documentName`
+	IssueDate time.Time `json:"issueDate"`
+	Modified time.Time `json:"modified"`
+	Expires time.Time `json:"expires"`
+	State string `json:"state"`
+	Author string `json:"author"`
+	Length int64 `json:"length"`
+	Hash hash
+
+	Description string `json:"description"`
+	Url string `json:"url"`
+	Version string `json:"version"`
+	Content_Type string `json:"content-type"`
+	Encoding string `json:"encoding"`
+	Compression string `json:"compression"`
+	Encryption string `json:"encryption"`
+
+	Attributes []Attribute `json:"attributes"`
+}
+
+func NewResource(config configuration) Resource {
+
+	r := Resource {}
+	u := uuid.New()
+	r.Uid = u.String()
+
+	r.IssueDate = time.Now()
+	r.State = config.state
+	r.Author  = config.author
+	r.Expires = config.ExpireDate()
+	r.Encoding = config.encoding
+
+	return r
+}
+
+func NewMartiLQResource(config configuration, sourcePath string, urlPath string, excludeHash bool, extendAttributes bool) (Resource, error) {
+	
+	r := Resource {}
+
+	stats, err := os.Stat(sourcePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Printf("'" + sourcePath + "' file does not exist.")
+			return r, errors.New("'" + sourcePath + "' file does not exist.")
+		}
+	}
+
+	u := uuid.New()
+	r.Uid = u.String()
+
+	r.State = config.state
+	r.Author  = config.author
+	r.Expires = config.ExpireDate()
+	r.Encoding = config.encoding
+
+	r.DocumentName = stats.Name()
+	if config.title == "documentName" {
+		r.Title = r.DocumentName
+	}
+	r.IssueDate = time.Now()
+	r.Modified = stats.ModTime()
+	r.Url = urlPath
+	r.Length = stats.Size()
+	if !excludeHash {
+		h := NewMartiLQHash(config.hashAlgorithm, sourcePath, "", config.signKey_File)
+		r.Hash = h
+	}
+
+	r.Attributes = NewDefaultExtensionAttributes(sourcePath, extendAttributes)
+
+	return r, nil
+}

source/golang/client/src/martilq/resource_test.go

@@ -0,0 +1,19 @@
+package martilq
+
+import (
+	"testing"
+	"strconv"
+)
+
+func TestResource_Default(t *testing.T) {
+	c := NewConfiguration()
+	r, err := NewMartiLQResource(c, "./resource.go", "", false, true)
+
+	if err != nil {
+		t.Error("Error returned")
+	}
+
+	if len(r.Attributes) != 1 {
+		t.Error("Arrays size not 1: " + strconv.Itoa(len(r.Attributes)))
+	}
+}

source/martilq.ini

@@ -0,0 +1,39 @@
+
+[General]
+
+logPath = 
+
+
+[MartiLQ]
+
+tags         = 
+publisher    = 
+contactPoint = 
+accessLevel  = Confidential
+rights       = None
+license      = 
+batch        = 
+theme        = 
+
+
+[Resources]
+
+author   = 
+title    = documentName
+state    = active
+expires  = 2:0:0
+encoding = UTF-8
+version  = 
+
+[Hash]
+
+hashAlgorithm =
+signKey_File =
+signKey_Password =
+
+
+[Network]
+
+proxy = 
+username =
+password = 

source/powershell/MartiLQ.ps1

@@ -81,6 +81,7 @@ function New-MartiDefinition
     [System.Collections.ArrayList]$lresource = @()
 
     $oMarti = [PSCustomObject]@{
+        "content-type" = "application/vnd.martilq.json"
         title = ""
         uid = (New-Guid).ToString()
         resources = $lresource

source/powershell/MartiLQItem.ps1

@@ -34,6 +34,7 @@ Param(
         }
 
         $lattribute =  Set-MartiResourceAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
+        $expires = (Get-Date).AddYears(7)
 
         $oResource = [PSCustomObject]@{ 
             title = $item.Name.Replace($item.Extension, "")
@@ -41,6 +42,7 @@ Param(
             documentName = $item.Name
             issuedDate = Get-Date -f "yyyy-MM-ddTHH:mm:ss"
             modified = $item.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")
+            expires = $expires -f "yyyy-MM-ddTHH:mm:ss"
             state = "active"
             author = ""
             length = $item.Length

source/python/client/martiLQ.py

@@ -85,8 +85,16 @@ class martiLQ:
               self.WriteLog("Usig configuration path '{}'".format(ConfigPath))
               config_object.read(ConfigPath)
 
+        if config_object.has_section("General"):
+            items = config_object["General"]
+            if not items is None:
+                config_attr = ["logPath"]
+                for x in config_attr:
+                    if not items[x] is None and not items[x] == "":
+                        self._oConfiguration[x] = items[x]
+
         if config_object.has_section("Resources"):
-            items = config_object["Resource"]
+            items = config_object["Resources"]
             if not items is None:
                 config_attr = ["accessLevel", "rights", "state"]
                 for x in config_attr:
@@ -255,6 +263,25 @@ class martiLQ:
 
         return self._oMartiDefinition
 
+    def Temporal(self):
+
+        oTemporal = {
+            "extension": "temporal",
+            "businessDate": "",
+            "runDate": ""
+        }
+
+        return oTemporal
+
+    def Spatial(self):
+
+        oSpatial = {
+            "country": "",
+            "region": "",
+            "town": "",
+        }
+
+        return oSpatial
 
     def NewMartiChildItem(self, SourceFolder, UrlPath=None, Recurse=True, ExtendAttributes=True, ExcludeHash=False, Filter ="*"):
 
@@ -327,6 +354,7 @@ class martiLQ:
                 "documentName": item,
                 "issuedDate": dateToday,
                 "modified": last_modified_date,
+                "expires": "",
                 "state": self.GetConfig("state"),
                 "author": self.GetConfig("author"),
                 "length": os.path.getsize(SourcePath),
@@ -336,6 +364,7 @@ class martiLQ:
                 "url": "",
                 "version": "",
                 "content-type": self.GetContentType(SourcePath),
+                "encoding": None,
                 "compression": None,
                 "encryption": None,
 
@@ -436,18 +465,14 @@ class martiLQ:
 
     def SetMartiAttribute(self, Attributes, ACategory, AName, AFunction, Comparison, Value):
 
-        matched = False
-    
         for attr in Attributes:
 
             if attr["category"] == ACategory and attr["name"] == AName and attr["function"] == AFunction:
-                matched = True
                 attr["comparison"] = Comparison
                 attr["value"] = Value
+                return Attributes
             
 
-        if not matched:
-            
             oAttribute = {
                 "category": ACategory,
                 "name": AName,