updates to Go and Python functions

source/golang/client/src/martilq/config.go

@@ -12,12 +12,12 @@ import (
 
 const cSoftwareName = "MARTILQREFERENCE"
 const cSoftwareAuthor = "Meerkat@merebox.com"
-const cSoftwareVersion = "0.0.1"
+const cSoftwareVersion = "0.0.2"
 const cIniFileName = "martilq.ini"
 const cExpires = "t:7:0:0"
 const cEncoding = ""
 
-type configuration struct {
+type Configuration struct {
 	softwareName string
 
 	dateFormat string
@@ -68,14 +68,16 @@ func GetSoftwareName() string {
 	return cSoftwareName
 }
 
-func NewConfiguration() configuration {
+func NewConfiguration() Configuration {
 
-	c :=  configuration {}
+	c :=  Configuration {}
 
 	c.softwareName = GetSoftwareName()
 
 	c.dateFormat = "2006-01-02"
 	c.dateTimeFormat = "2006-01-02T15:04:05-0700"
+	c.dataPath = ""
+	c.tempPath =  "temp"
 
 	c.title = "{{documentName}}"
 	c.state = "active"
@@ -95,6 +97,8 @@ func NewConfiguration() configuration {
 	c.spatial = GetSpatial()
 	c.temporal = GetTemporal()
 
+
+
 	configPath := findIni()
 	if configPath != "" {
 		c.LoadConfig(configPath)
@@ -145,7 +149,7 @@ func findIni() string {
 	return foundPath
 }
 
-func (c *configuration) SaveConfig(ConfigPath string) bool {
+func (c *Configuration) SaveConfig(ConfigPath string) bool {
 
 	cfgini, _ := ini.LooseLoad("./martilq.ini")
 		
@@ -197,7 +201,7 @@ func (c *configuration) SaveConfig(ConfigPath string) bool {
 	return true
 }
 
-func (c *configuration) LoadConfig(ConfigPath string) bool {
+func (c *Configuration) LoadConfig(ConfigPath string) bool {
 
 	if ConfigPath != "" {
 		_, err := os.Stat(ConfigPath)
@@ -286,7 +290,7 @@ func Loadenv(key string, default_value string ) string {
 }
 
 
-func (c *configuration) ExpireDate(sourcePath string ) time.Time {
+func (c *Configuration) ExpireDate(sourcePath string ) time.Time {
 
 	var expires time.Time
 

source/golang/client/src/martilq/logging.go

@@ -1,10 +0,0 @@
-package martilq
-
-
-import (
-	"log"
-)
-
-func WriteLog(entry string) {
-	log.Printf(entry)
-}

source/golang/client/src/martilq/marti.go

@@ -44,7 +44,7 @@ type Marti struct {
 	Custom []interface{} `json:"custom"`
 
 
-	config configuration
+	config Configuration
 }
 
 func NewMarti() Marti {
@@ -63,7 +63,7 @@ func NewMarti() Marti {
 	return m
 }
 
-func Load(c configuration, pathFile string) (Marti, error) {
+func Load(c Configuration, pathFile string) (Marti, error) {
 	
 	m := Marti {}
 

source/golang/client/src/martilq/resource.go

@@ -40,7 +40,7 @@ type Resource struct {
 	Attributes []Attribute `json:"attributes"`
 }
 
-func NewResource(config configuration) Resource {
+func NewResource(config Configuration) Resource {
 
 	r := Resource {}
 	u := uuid.New()
@@ -57,7 +57,7 @@ func NewResource(config configuration) Resource {
 	return r
 }
 
-func NewMartiLQResource(config configuration, sourcePath string, urlPath string, excludeHash bool, extendAttributes bool) (Resource, error) {
+func NewMartiLQResource(config Configuration, sourcePath string, urlPath string, excludeHash bool, extendAttributes bool) (Resource, error) {
 	
 	r := Resource {}
 

source/golang/client/src/martilq/utility.go

@@ -1,2 +1,93 @@
 package martilq
 
+import (
+	"os"
+	"path"
+	"time"
+	"strings"
+	"log"
+	"errors"
+)
+
+func GetLogName() string {
+
+    date := time.Now().Format("2006-01-02")
+	logPathName := "./logs"
+    
+    if (logPathName == "") {
+        return "./logs"
+    }
+	
+	if _, err := os.Stat(logPathName); errors.Is(err, os.ErrNotExist) {
+		err := os.Mkdir(logPathName, os.ModePerm)
+		if err != nil {
+			log.Println(err)
+		}
+	}
+
+    logName := GetSoftwareName() + "_" + date + ".log"
+    sFullPath := path.Join(logPathName, logName )
+	
+	_, err := os.Stat(sFullPath)
+  	if err != nil && os.IsNotExist(err) {
+		log.Println("Log path: "+ sFullPath)
+	}
+
+    return sFullPath
+}
+
+
+func WriteLog(LogEntry string) {
+    
+    sFullPath := GetLogName() 
+	if (sFullPath != "") {
+		logFile, err := os.OpenFile(sFullPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer logFile.Close()
+		log.SetOutput(logFile)
+    }
+	log.Println(LogEntry)
+}
+
+func OpenLog() {
+    dateTime := time.Now().Format("2006-01-02T15:04:05-0700")
+    WriteLog( "***********************************************************************************")
+    WriteLog( "*   Start of processing: ["+dateTime+"]")
+    WriteLog( "***********************************************************************************")
+}
+
+func CloseLog() {
+    dateTime := time.Now().Format("2006-01-02T15:04:05-0700")
+    WriteLog( "***********************************************************************************")
+    WriteLog( "*   End of processing: ["+dateTime+"]")
+    WriteLog( "***********************************************************************************")
+}
+
+
+func NewLocalTempFile(UrlPath string, configuration *Configuration, TempPath string) string {
+
+    parts := strings.Split(UrlPath, "/")
+    doc_name := parts[len(parts)-1]
+	temp_dir := TempPath
+
+	if (temp_dir == "") {
+		if (configuration == nil) {
+			temp_dir = NewConfiguration().tempPath
+		} else {
+			temp_dir = configuration.tempPath
+		}
+	}
+
+	if _, err := os.Stat(temp_dir); errors.Is(err, os.ErrNotExist) {
+		err := os.Mkdir(temp_dir, os.ModePerm)
+		if err != nil {
+			log.Println(err)
+		} else {
+        	WriteLog("Created temp folder : " + temp_dir)
+		}
+	}
+
+    return path.Join(temp_dir, doc_name )
+}

source/golang/client/src/tool.go

@@ -142,13 +142,13 @@ func loadArguments(args []string) {
 					params.description = args[ix]
 				}
 			} else {
-				panic("Missing parameter for DECRIPTION")
+				panic("Missing parameter for DESCRIPTION")
 			}
 		}
 
 
 		if !matched && args[ix] != "--" {
-			fmt.Println("Unrecognised command line argument: " + args[ix])
+			fmt.Println("Unrecognized command line argument: " + args[ix])
 		}
 
 		ix = ix + 1
@@ -160,7 +160,7 @@ func printHelp() {
 
 	fmt.Println("")
 	fmt.Println("\t   martilqcli_client ")
-	fmt.Println("\t   =======++======== ")
+	fmt.Println("\t   ================= ")
 	fmt.Println("")
 	fmt.Println("\tThis program is intended as a simple reference implementation")
 	fmt.Println("\tin Go of the MartiLQ framework.  It is does not provide all")
@@ -171,10 +171,10 @@ func printHelp() {
 	fmt.Println("")
 	fmt.Println(" -h or --help : Display this help")
 	fmt.Println(" -t or --task : Execute a predefined task which are")
-	fmt.Println("           INIT initialise a new configuration file")
+	fmt.Println("           INIT initialize a new configuration file")
 	fmt.Println("           MAKE make a MartiLQ definition file")
 	fmt.Println("           GET resources based on MartiLQ definition file")
-	fmt.Println("           RECON reconicile a MartiLQ definition file")
+	fmt.Println("           RECON reconcile a MartiLQ definition file")
 	fmt.Println(" -c or --config : Configuration file used by all tasks")
 	fmt.Println("           This is the file written by the INIT task")
 	fmt.Println(" -s or --source : Source directory or file to build MartiLQ definition")
@@ -218,62 +218,13 @@ func main () {
 	} else {
 
 
-		if params.task == "INIT" {
-			if params.configPath == "" {
-				panic("Missing 'config' parameter")
-			}
-
-			_, err := os.Stat(params.configPath)
-			if err == nil {
-				panic("MartiLQ configuration file '"+ params.configPath+"' already exists")
-			}
-
-			c := martilq.NewConfiguration()
-			if c.SaveConfig(params.configPath) != true {
-				panic("Configuration not saved to: "+ params.configPath)
-			}
-			fmt.Println("Created MARTILQ INI definition: " + params.configPath)
-			matched = true
-		}
-
-		if params.task == "MAKE" {
-
-			if params.sourcePath == "" {
-				panic("Missing 'source' parameter")
-			}
-			if params.definitionPath == "" {
-				panic("Missing 'output' parameter")
-			}
-
-			_, err := os.Stat(params.definitionPath)
-			if err == nil && params.update == false {
-				panic("MartiLQ document '"+ params.definitionPath+"' already exists and update not specified")
-			}
-
-			m := martilq.Make(params.configPath, params.sourcePath, params.filter, params.recursive, params.urlPrefix, params.definitionPath )
-			if params.title != "" {
-				m.Title = params.title
-			}
-			if params.description != "" {
-				m.Description = params.description
-			}
-			m.Save(params.definitionPath)
-
-			fmt.Println("Created MARTILQ definition: " + params.definitionPath)
-			matched = true
-		}
-
-		if params.task == "GET" {
-			fmt.Println("ET task not implemented")
-			matched = true
-		}
-
-
 		if params.task == "RECON" {
 
-			_ = martilq.ReconcileFilePath(params.configPath, params.sourcePath, params.recursive, params.definitionPath, params.outputPath )
+			m = martilq.ReconcileFilePath(params.configPath, params.sourcePath, params.recursive, params.definitionPath, params.outputPath )
-			
+		
 			matched = true
+
+
 		}
 
 		if !matched {

source/golang/server/src/docs/acknowledgement.md

@@ -0,0 +1,68 @@
+# Acknowledgment
+
+Once the **martiLQ** document is received by a consumer then communicating the receipt, processing,
+success or failure completes the feedback loop and builds an extra layer of assurance for the organization.
+
+The acknowledgement workflow provides the necessary feedback.  If an acknowledgement is required as part of the 
+consumption design then the following is approach is recommended.
+
+1. The publisher provides callback details.  For extra security the callback  details should be signed.
+2. The consumer will acknowledge the receipt of the **martiLQ** document by sending back the same
+   document to the publisher with some values changed.
+3. Change the root consumer and state (not resource) from ``active`` to ``receipt`` and modify the ``stateModified`` timestamp.
+4. Change the ``consumer`` data value to only be your identifier and not others, so that the publisher
+   can identify the consumer and associate it with success or failure.  This change to consumer value
+   applies to all subsequent acknowledgement messages. 
+5. Send the changed **martiLQ** document back using the callback details
+6. On fetching each resource the resource state is changed from ``active`` to ``received`` and modify the ``stateModified`` timestamp.  
+   If any resource cannot be retrieved the state is changed from ``active`` to ``missing`` and ``stateModified`` timestamp is udpated.
+7. The consumer can elect to send back the **martiLQ** document to the publisher on each fetch or at the completion
+   of all fetches.  The recommendation is to send at the end of all fetches because if there are issues then
+   having all the failures for analysis should assist in determining the extent of the failure.
+8. Once all resources are fetched (or failed), the root state is changed from ``receipt`` to ``received`` and update the ``stateModified`` 
+   timestamp if no errors occurred in retrieving the resources. If a single or many errors occurred, then the root state is
+   changed from ``receipt`` to ``missing`` and the ``stateModified`` timestamp is updated.  The updated document is sent back 
+   to the publisher using the callback details.
+9. The next stage is to validate and process the resources defined in the **martiLQ** document.  This follows
+   a similar process to fetching the resources.
+10. On processing each resource the resource state is changed from ``received`` to ``processed`` and modify the ``stateModified`` timestamp.  
+   If any resource cannot be processed the state is changed from ``received`` to ``error`` and update the ``stateModified`` timestamp.  Once 
+   again this can be acknowledged back to the publisher.
+11. Once all resources are processed (or failed), the root state is changed from ``received`` to ``processed`` and the ``stateModified`` timestamp
+   updated if no errors occurred in processing the resources. If a single or many errors occurred, then the root state is
+   changed from ``received`` to ``error`` and the ``stateModified`` timestamp modified.  The updated document is sent back to the publisher using 
+   the callback details.
+
+This completes the acknowledgment workflow for the **martiLQ** document.  The level of acknowledgement feedback
+you wish to implement as a consumer is your decision.  Any publisher providing callback details for acknowledgement can also
+choose their behavior on actions and recording any acknowledgments received.
+
+In the above acknowledgement process, you **must not** change the identifiers in the **martiLQ** document and you **should not**
+change other data except the ``consumer`` and ``state`` and ``stateModified``.
+
+If you are the publisher and expect acknowledgment then there is an extra scenario you need to cater for.  The scenario is
+that you do not receive any acknowledgement back from the expected consumer(s) within the agreed time frame.  In this situation
+the publisher will need to know each consumer and their service level agreements.
+
+## Callback
+
+The callback method can take any form.  The normal expectation is that the same method used to communicate the **martiLQ** 
+document is used for the callback for example:
+
+* If the **martiLQ** document is originally sent by Kafka queue then the callback should use a Kafka queue separate topic
+* If the **martiLQ** document is originally sent by REST API then the callback is a publisher REST API end point
+* If the **martiLQ** document is originally sent by email then the callback uses a reply address for the callback
+
+While the above is the expected convention, you can mix and match.  For example:
+
+* If the **martiLQ** document is originally sent by REST API then the callback can be an email address.  This 
+  situation could be acceptable if the publisher does not have the ability to accept REST API call backs.
+
+## Compressed file handling
+
+When the **martiLQ** document is defining a parent compressed file, e.g. ZIP or 7Z, then the resources are expected
+to be in the compressed file.  These resources can still be checked for existence and that they can be extracted.  The
+state of the resource is still changed to reflect the processing.
+
+If the file cannot be extracted either because it has not been included or there is a decompression error, then the
+same acknowledgement process of using the state is used.

source/golang/server/src/docs/comparison.md

@@ -1,11 +1,10 @@
-Comparison of martiLQ document definition
+# Comparison of martiLQ document definition
-=========================================
 
 The use of metadata definitions is not unique and examples
 exist in many different situations.  Some are standard and open
 while others are closed.
 
-Some open standards are EXIF data for pictures, SQL DDL defintions
+Some open standards are EXIF data for pictures, SQL DDL definitions
 for databases, the XMP definition and web header responses before the 
 web content.
 
@@ -23,18 +22,16 @@ a CKAN source into a **martiLQ** document definition and then process the data
 through the pipeline as you would for any other data file that had a
 **martiLQ** document definition.
 
-Benefit of CKAN and martiLQ
+## Benefit of CKAN and martiLQ
----------------------------
 
 The CKAN is excellent at defining the data source details but it lacks information
-for load quality.  If you have CKAN deployed in your organisation and wish
+for load quality.  If you have CKAN deployed in your organization and wish
 exhange or process the data referenced in CKAN, then there are synergies between
 CKAN and marti.
 
 Samples exist on CKAN integration.
 
-Magda and martiLQ
+## Magda and martiLQ
------------------
 
 Another source of data is [Magda](https://magda.io/) which has API metadata
 definitions.  Magda is more about data federation and as such provides

source/golang/server/src/docs/custom.md

@@ -1,7 +1,7 @@
 Custom Definition
 =================
 
-The custome definition section allows the inclusion of extensions
+The custom definition section allows the inclusion of extensions
 to the standard.  To demonstrate the inclusion, there are three
 sample extensions.  These are:
 

source/golang/server/src/docs/magda.md

@@ -1,5 +1,5 @@
-Magda definitions
+# Magda definitions
-=================
+
 
 https://magda.io/ 
 

source/golang/server/src/docs/martiLQ.md

@@ -39,7 +39,7 @@ modified|Modified date and time of the **martiLQ** document|Now
 tags|List of tags or keywords|
 publisher|Publisher name|
 contactPoint|Contact point of a person or team|
-accessLevel|Acces level|
+accessLevel|Access level|
 rights|Rights|
 
 * Batch
@@ -54,7 +54,7 @@ rights|Rights|
 ### Information extension
 
 The information supplied can be extended by party agreement and there
-are place holders in the defintion.
+are place holders in the definition.
 
 ## Resource 
 
@@ -62,7 +62,7 @@ The resource section is a list of documents or files that are to be grouped
 together are listed under the same **martiLQ** definition.
 
 At least one document or file must be included.  If the same resource is repeated 
-it will commonly be for definiting multiple formats, with each file having a 
+it will commonly be defining multiple formats, with each file having a 
 different extension.  Commonly the definition includes at least the following
 items:
 
@@ -84,7 +84,7 @@ for more details
 Name|Description|Default or values
 ---|---|--
 hash|Hash of document - The hash of the document, which can be blank especially for large documents
-algo|Hash algorithm - Algoroithm used to generate the hash value or sign it
+algo|Hash algorithm - Algorithm used to generate the hash value or sign it
 description|Description - A more detailed description 
 version|Version - A document version
 encoding|Encoding
@@ -117,6 +117,7 @@ sample can be generated using the GOLANG client program with parameters:
     "tags": null,
     "license": "",
     "state": "active",
+    "stateModified": "2021-11-02T22:44:29.6887001+11:00",
     "batch": 1.001,
     "describedBy": "",
     "landingPage": "",
@@ -130,6 +131,7 @@ sample can be generated using the GOLANG client program with parameters:
             "modified": "2021-11-02T07:47:13.9410018+11:00",
             "expires": "2023-11-02T00:00:00+11:00",
             "state": "active",
+            "stateModified": "2021-11-02T22:44:29.6881663+11:00",
             "author": "",
             "length": 3654,
             "hash": {

source/golang/server/src/docs/quality.md

@@ -19,7 +19,7 @@ load quality metrics.
 * Number of records in the document - This is the number of data primary records not the 
    count of end of lines and is agreed between parties.  XML record counts could be based 
    on the number of primary segments under root.  JSON records can be counted in a similar way.
-   The headers or trailling records are not counted
+   The headers or trailing records are not counted
 
 ## Addresses deficiencies 
 

source/golang/server/src/docs/references.md

@@ -1,7 +1,7 @@
 # References
 
 The following are references to documents that inspired the creation of **martiLQ**
-document and associatd framework.
+document and associated framework.
 
 https://dex.dss.gov.au/sites/default/files/documents/2021-06/data-exchange-protocols-june-2021.pdf
 

source/golang/server/src/docs/what.md

@@ -12,7 +12,7 @@ for various programming languages and situations.  As many programming languages
 generate portable programs that can execute on multiple operating systems, the
 likelihood is that a tools exists for you.
 
-The source for tools is provided in the Github repository and some have precompiled
+The source for tools is provided in the Github repository and some have pre-compiled
 images.
 
 See the project source directory for more details.

source/golang/server/src/docs/who.md

@@ -1,13 +1,11 @@
-Who is likely to use martiLQ
+# Who is likely to use martiLQ
-============================
 
 You are likely to find the **martiLQ** framework relevant if you:
 
 1. Have many document exchanges, such as End of Day batches
 2. Need to verify or reconcile the documents
 
-Data exchanges
+## Data exchanges
---------------
 
 If you are creating or receiving many documents or files on a regular basis
 then you probably have some framework defined.  The framework may be as simple as:
@@ -24,8 +22,7 @@ Simple framework such as the above have limitations, such as:
 * Lower automation prospects and alignment to DataSecOps
 * Poor fit to web applications (they tend to be designed for FTP and LAN)
 
-Framework Sidecar files
+## Framework Sidecar files
------------------------
 
 The **martiLQ** framework addresses the issues and limitations by using sidecar
 or shadow files. The [concept of sidecar files](https://en.wikipedia.org/wiki/Sidecar_file) is 
@@ -35,7 +32,7 @@ Sidecar files can also be implemented as ``forks`` and built into the operating
 in Mac OS X HFS.  The Microsoft NTFS supports Alternate Data Streams to achieve a similar outcome.
 Unfortunately this information is not transferrable to other systems. 
 
-The proposition is to define a format for the sidecare file and provide common library tools that
+The proposition is to define a format for the sidecar file and provide common library tools that
 can be be used on multiple platforms when exchanging documents / files.  Multiple documents can be 
-defined in a singel **martiLQ** definition which adds to efficiency and productivity if used 
+defined in a single **martiLQ** definition which adds to efficiency and productivity if used 
 for End of Day or similar batches - or even single file transfers.

source/martilq.ini

@@ -37,3 +37,10 @@ signKey_Password =
 proxy = 
 username =
 password = 
+
+[Smtp]
+
+host = 
+port =
+username =
+password = 

source/powershell/MartiLQ.ps1

@@ -5,7 +5,7 @@
 . .\source\powershell\MartiLQAttribute.ps1
 
 
-function New-MartiDefinition
+function New-MartiLQDefinition
 {
     Param( 
         [String] $ConfigPath = $null    
@@ -23,13 +23,6 @@ function New-MartiDefinition
         renderer =  "MARTILQREFERENCE:Mustache"
         url = ""
     }
-   
-    $oAcknowledgement = [PSCustomObject]@{
-        url = ""
-        algo = ""
-        value = ""
-        signed =  $false
-    }
 
 
     if ($null -eq $ConfigPath -or $ConfigPath -eq "") {
@@ -97,7 +90,7 @@ function New-MartiDefinition
         publisher = $publisher
         contactPoint = $oConfig.contactPoint
         accessLevel = $oConfig.accessLevel
-        consumer = $lconsumer
+        consumers = $lconsumer
         rights = $oConfig.rights
         license = $oConfig.license
         state = $oConfig.state
@@ -108,7 +101,7 @@ function New-MartiDefinition
         theme =$oConfig.theme
 
         resources = $lresource
-        acknowledge = $oAcknowledgement
+        acknowledge = Get-Acknowledgement
         custom = $lCustom
     }
 
@@ -116,6 +109,94 @@ function New-MartiDefinition
 }
 
 
+function Save-MartiLQDefinition
+{
+    Param( 
+        [Parameter(Mandatory)][PSCustomObject] $MartiLQ,
+        [Parameter(Mandatory)][String] $FilePath
+    ) 
+
+    $fileJson = $FilePath
+    $MartiLQ | ConvertTo-Json -depth 100 | Out-File $fileJson
+
+    return $fileJson
+}
+
+
+function Restore-MartiLQDefinition
+{
+    Param( 
+        [Parameter(Mandatory)][String] $FilePath
+    ) 
+
+    $oMartiLQ = [PSCustomObject](Get-Content -Raw $FilePath | Out-String | ConvertFrom-Json)
+    
+    $version = Get-DefinitionVersion -MartiLQ $oMartiLQ
+    if ($version -lt "0.0.2") {
+        if (![bool]($oMartiLQ.PSCustomObject.Properties.name -match "consumers")) {
+            [System.Collections.ArrayList]$lconsumer = @()
+            $oMartiLQ | Add-Member -Name "consumers" -Type NoteProperty -Value $lconsumer
+        }
+
+        if (![bool]($oMartiLQ.PSCustomobject.Properties.name -match "acknowledge")) {
+            $oMartiLQ | Add-Member -Name "acknowledge" -Type NoteProperty -Value (Get-Acknowledgement)
+        }
+
+        Set-DefinitionVersion -MartiLQ $oMartiLQ -Version "0.0.2"
+        $newVersion = Get-DefinitionVersion -MartiLQ $oMartiLQ
+        Write-Host "Updating from version $version to $newVersion"
+    }
+
+    return $oMartiLQ
+}
+
+function Get-DefinitionVersion
+{
+param (
+    [Parameter(Mandatory)][PSCustomObject] $MartiLQ
+)
+   
+    $version = "0.0.1"
+    $MartiLQ.custom | ForEach-Object {
+        if ($_.extension -eq "software" -and $_.softwareName -eq (Get-SoftwareName)) {
+            $version = $_.version
+        }        
+    }
+
+    return $version
+}
+
+function Set-DefinitionVersion
+{
+param (
+    [Parameter(Mandatory)][PSCustomObject] $MartiLQ,
+    [Parameter(Mandatory)][String] $Version
+)
+   
+    $MartiLQ.custom | ForEach-Object {
+
+        if ($_.extension -eq "software" -and $_.softwareName -eq (Get-SoftwareName)) {
+                $_.version = $Version
+                #return $MartiLQ
+        }
+        
+    }
+
+    #return $MartiLQ
+}
+
+function Get-Acknowledgement{
+   
+    $oAcknowledgement = [PSCustomObject]@{
+        url = ""
+        algo = ""
+        value = ""
+        signed =  $false
+    }
+
+    return $oAcknowledgement
+}
+
 function Set-MartiAttribute
 {
 Param( 
@@ -244,9 +325,6 @@ function Get-MartiResource {
 }
 
 
-
-
-
 function ConvertFrom-Ckan 
 {
 Param( 
@@ -341,7 +419,6 @@ Param(
 }
 
 
-
 function Compress-MartiLQ
 {
 Param( 
@@ -397,6 +474,3 @@ Param(
     Close-Log
 }
 
-
-
-

source/powershell/MartiLQConfiguration.ps1

@@ -1,7 +1,7 @@
 
 
 
-$script:SoftwareVersion = "0.0.1"
+$script:SoftwareVersion = "0.0.2"
 $global:default_metaFile = "##martilq##.json"
 
 $global:martiLQ_configuration = $null
@@ -58,6 +58,11 @@ function Get-DefaultConfiguration {
         proxy_User = $null
         proxy_Credential = $null
 
+        smtp_Host = $null
+        smtp_Port = $null
+        smtp_User = $null
+        smtp_Credential = $null
+
         loaded = $false
     }
 
@@ -155,6 +160,13 @@ function Import-Configuration {
             $oConfig.hashAlgorithm = Set-ConfigurationValue $oConfig.hashAlgorithm -Value $iConfig.Hash.hashAlgorithm
         }
 
+        if ($null -ne $iConfig.Smtp) {
+            $oConfig.smtp_Host = Set-ConfigurationValue $oConfig.smtp_Host -Value $iConfig.Smtp.host
+            $oConfig.smtp_Port = Set-ConfigurationValue $oConfig.smtp_Port -Value $iConfig.Smtp.port
+            $oConfig.smtp_User = Set-ConfigurationValue $oConfig.smtp_Port -Value $iConfig.Smtp.username
+            $oConfig.smtp_Credential = Set-ConfigurationValue $oConfig.smtp_Credential -Value $iConfig.Smtp.password
+        }
+
     }
 
     $global:martiLQ_configuration = $oConfig

source/powershell/MartiLQTool.ps1

@@ -0,0 +1,186 @@
+
+. .\source\powershell\MartiLQ.ps1
+. .\source\powershell\MartiLQConfiguration.ps1
+
+
+function Send-EmailAck {
+    param (
+        [String] $FileAttachment,
+        [String] $Recipient,
+        [String] $State,
+        [int] $Buffersize = 1024
+    )
+
+    $receiver = $Recipient.Substring(5)
+    Write-Host "Sending acknowledgment via email to: $receiver " -ForegroundColor Green
+  
+    $EmailFrom = $env:MARTILQ_EMAIL_FROM
+    
+    $Subject = "martiLQ acknowledge [$State]"
+    $Body = "Simple email ack"
+
+    $password = ConvertTo-SecureString $env:MARTILQ_EMAIL_PASSWORD -AsPlainText -Force
+    $credential = New-Object System.Management.Automation.PSCredential ($env:MARTILQ_EMAIL_USERNAME, $password)
+    
+    Write-Host "SMTP: $($env:MARTILQ_EMAIL_HOST) :: 465 " -ForegroundColor Yellow
+    Write-Host "Send with: $FileAttachment :: $Subject  :: $Body " -ForegroundColor Yellow
+
+    $att = new-object Net.Mail.Attachment($FileAttachment)
+
+    #Send mail with attachment
+    $from = New-Object System.Net.Mail.MailAddress($EmailFrom)
+    $to = New-Object System.Net.Mail.MailAddress($receiver)
+    $email = New-Object System.Net.Mail.Mailmessage($from, $to)
+    $email.Subject = $Subject
+    $email.Body = $Body
+    $email.IsBodyHTML = $true
+    $email.Attachments.Add($att)
+   
+    $att.Dispose()
+
+    $smtp = New-Object Net.Mail.SmtpClient($env:MARTILQ_EMAIL_HOST, 465)
+    $smtp.EnableSSL = $true
+    $smtp.Credentials = New-Object System.Net.NetworkCredential($env:MARTILQ_EMAIL_USERNAME, $env:MARTILQ_EMAIL_PASSWORD)
+    $smtp.Send($email)
+ 
+}
+
+function Get-Resources {
+    param (
+        [Parameter(Mandatory)][PSCustomObject] $MartiLQ,
+        [String] $DataPath,
+        [String] $CurrentState,
+        [String] $Consumer
+    )
+
+    $nextState = "received"
+    
+    Try {
+        foreach ($item in $MartiLQ.resources) {
+            if ($item.state -eq $currentState) {
+
+                $item.state = $nextState
+            }
+        }
+
+        if ($Consumer -ne "") {
+            [System.Collections.ArrayList]$lconsumer = @()
+            $lconsumer += $Consumer
+            $MartiLQ.consumers = $lconsumer
+        }
+
+        $today = Get-Date
+        $dateToday = $today.Tostring("yyyy-MM-ddTHH:mm:ss")
+        $MartiLQ.stateModified = $dateToday
+        $MartiLQ.state = $nextState
+
+        # Notification
+        $ack = $MartiLQ.acknowledge
+
+        if ($ack.url.startswith("mail:"))
+        {
+            $fileJson = "./temp/MartiBSBRemote_interim2.json"
+            $attachment = Save-MartiLQDefinition -MartiLQ $oMarti -FilePath $fileJson
+            Send-EmailAck -FileAttachment $attachment -Recipient $ack.url -State $nextState
+        }
+
+    } Catch {
+        Write-Host "Error in resource get: $_"
+    }
+
+    return $MartiLQ
+}
+
+
+function Test-Resource {
+    param (
+        [Parameter(Mandatory)][PSCustomObject] $MartiLQ,
+        [String] $DataPath,
+        [String] $CurrentState,
+        [String] $Consumer
+    )
+
+    $nextState = "verified"
+    
+    Try {
+        foreach ($item in $MartiLQ.resources) {
+            if ($item.state -eq $currentState) {
+
+                $item.state = $nextState
+            }
+        }
+
+        if ($Consumer -ne "") {
+            [System.Collections.ArrayList]$lconsumer = @()
+            $lconsumer += $Consumer
+            $MartiLQ.consumers = $lconsumer
+        }
+
+        $MartiLQ.state = $nextState
+    } Catch {
+        Write-Host "Error in resource test: $_"
+    }
+
+    return $MartiLQ
+}
+
+function Invoke-ProcessResource {
+    param (
+        [Parameter(Mandatory)][PSCustomObject] $MartiLQ,
+        [String] $DataPath,
+        [String] $CurrentState,
+        [String] $Consumer
+    )
+
+    $nextState = "processed"
+    
+    Try {
+        foreach ($item in $MartiLQ.resources) {
+            if ($item.state -eq $currentState) {
+
+                $item.state = $nextState
+            }
+        }
+
+        if ($Consumer -ne "") {
+            [System.Collections.ArrayList]$lconsumer = @()
+            $lconsumer += $Consumer
+            $MartiLQ.consumers = $lconsumer
+        }
+
+        $MartiLQ.state = $nextState
+    } Catch {
+        Write-Host "Error in resource process: $_"
+    }
+
+    return $MartiLQ
+}
+
+$currentState = "expired"
+$nextState = "active"
+$consumer = "Test-Framework"
+
+$fileJson = "C:\Users\meerkat\source\marti\docs\source\samples\powershell\test\MartiBSBRemote.json"
+$oMarti = Restore-MartiLQDefinition -FilePath $fileJson
+
+$oMarti.acknowledge.url = "mail:tp_reklam@villacentrum.com"
+
+$today = Get-Date
+$dateToday = $today.Tostring("yyyy-MM-ddTHH:mm:ss")
+$oMarti.stateModified = $dateToday
+$oMarti.state = $nextState
+
+foreach ($item in $oMarti.resources) {
+    if ($item.state -eq $currentState) {
+        $item.state = $nextState
+    }
+}
+
+$oMarti = Get-Resources -MartiLQ $oMarti[0] -DataPath "" -CurrentState "active" -Consumer $consumer
+
+$oMarti = Test-Resource -MartiLQ $oMarti -DataPath "" -CurrentState "received" -Consumer $consumer
+
+$oMarti = Invoke-ProcessResource -MartiLQ $oMarti -DataPath "" -CurrentState "verified" -Consumer $consumer
+
+$fileJson = "C:\Users\meerkat\source\marti\docs\source\samples\powershell\test\MartiBSBRemote_v2.json"
+Save-MartiLQDefinition -MartiLQ $oMarti -FilePath $fileJson

source/powershell/MartiLQUtilities.ps1

@@ -91,4 +91,4 @@ function New-LocalTempFile{
     }
 
     return Join-Path -Path $temp_dir -ChildPath $doc_name
-}
+}

source/python/client/martiLQ.py

@@ -24,14 +24,13 @@ from mutility import mUtility
 
 class martiLQ:
 
-    _SoftwareVersion = "0.0.1"
     _default_metaFile = "##martilq##.json"
 
     _oSoftware = {
         "extension": "software",
-        "softwareName": "MARTILQREFERENCE",
+        "softwareName": mConfiguration.GetSoftwareName(),
         "author": "Meerkat@merebox.com",
-        "version": "0.0.1"
+        "version": mConfiguration.GetSoftwareVersion()
     }
 
     _oTemplate = {

source/python/client/mconfiguration.py

@@ -19,24 +19,26 @@ from mlogging import mLogging
 
 class mConfiguration:
 
-    _SoftwareVersion = "0.0.1"
+    _SoftwareVersion = "0.0.2"
     _default_metaFile = "##martilq##.json"
 
     _oSoftware = {
         "extension": "software",
-        "softwareName": "MARTILQREFERENCE",
+        "softwareName": GetSoftwareName(),
         "author": "Meerkat@merebox.com",
-        "version": "0.0.1"
+        "version": GetSoftwareVersion()
     }
     
-    
     _oConfiguration = None
     _Log = None
 
 
-    def GetSoftwareName(self):
+    def GetSoftwareName():
         return "MARTILQREFERENCE"
 
+    def GetSoftwareVersion():
+        return _SoftwareVersion
+
     def __init__(self):
 
         self._oConfiguration = {