diff --git a/source/golang/client/src/martilq/config.go b/source/golang/client/src/martilq/config.go index a940c26..37644a3 100644 --- a/source/golang/client/src/martilq/config.go +++ b/source/golang/client/src/martilq/config.go @@ -12,12 +12,12 @@ import ( const cSoftwareName = "MARTILQREFERENCE" const cSoftwareAuthor = "Meerkat@merebox.com" -const cSoftwareVersion = "0.0.1" +const cSoftwareVersion = "0.0.2" const cIniFileName = "martilq.ini" const cExpires = "t:7:0:0" const cEncoding = "" -type configuration struct { +type Configuration struct { softwareName string dateFormat string @@ -68,14 +68,16 @@ func GetSoftwareName() string { return cSoftwareName } -func NewConfiguration() configuration { +func NewConfiguration() Configuration { - c := configuration {} + c := Configuration {} c.softwareName = GetSoftwareName() c.dateFormat = "2006-01-02" c.dateTimeFormat = "2006-01-02T15:04:05-0700" + c.dataPath = "" + c.tempPath = "temp" c.title = "{{documentName}}" c.state = "active" @@ -95,6 +97,8 @@ func NewConfiguration() configuration { c.spatial = GetSpatial() c.temporal = GetTemporal() + + configPath := findIni() if configPath != "" { c.LoadConfig(configPath) @@ -145,7 +149,7 @@ func findIni() string { return foundPath } -func (c *configuration) SaveConfig(ConfigPath string) bool { +func (c *Configuration) SaveConfig(ConfigPath string) bool { cfgini, _ := ini.LooseLoad("./martilq.ini") @@ -197,7 +201,7 @@ func (c *configuration) SaveConfig(ConfigPath string) bool { return true } -func (c *configuration) LoadConfig(ConfigPath string) bool { +func (c *Configuration) LoadConfig(ConfigPath string) bool { if ConfigPath != "" { _, err := os.Stat(ConfigPath) @@ -286,7 +290,7 @@ func Loadenv(key string, default_value string ) string { } -func (c *configuration) ExpireDate(sourcePath string ) time.Time { +func (c *Configuration) ExpireDate(sourcePath string ) time.Time { var expires time.Time diff --git a/source/golang/client/src/martilq/logging.go b/source/golang/client/src/martilq/logging.go deleted file mode 100644 index e4d24e4..0000000 --- a/source/golang/client/src/martilq/logging.go +++ /dev/null @@ -1,10 +0,0 @@ -package martilq - - -import ( - "log" -) - -func WriteLog(entry string) { - log.Printf(entry) -} diff --git a/source/golang/client/src/martilq/marti.go b/source/golang/client/src/martilq/marti.go index 8d13680..a1d13d6 100644 --- a/source/golang/client/src/martilq/marti.go +++ b/source/golang/client/src/martilq/marti.go @@ -44,7 +44,7 @@ type Marti struct { Custom []interface{} `json:"custom"` - config configuration + config Configuration } func NewMarti() Marti { @@ -63,7 +63,7 @@ func NewMarti() Marti { return m } -func Load(c configuration, pathFile string) (Marti, error) { +func Load(c Configuration, pathFile string) (Marti, error) { m := Marti {} diff --git a/source/golang/client/src/martilq/resource.go b/source/golang/client/src/martilq/resource.go index b2f9eb8..0a1b8a2 100644 --- a/source/golang/client/src/martilq/resource.go +++ b/source/golang/client/src/martilq/resource.go @@ -40,7 +40,7 @@ type Resource struct { Attributes []Attribute `json:"attributes"` } -func NewResource(config configuration) Resource { +func NewResource(config Configuration) Resource { r := Resource {} u := uuid.New() @@ -57,7 +57,7 @@ func NewResource(config configuration) Resource { return r } -func NewMartiLQResource(config configuration, sourcePath string, urlPath string, excludeHash bool, extendAttributes bool) (Resource, error) { +func NewMartiLQResource(config Configuration, sourcePath string, urlPath string, excludeHash bool, extendAttributes bool) (Resource, error) { r := Resource {} diff --git a/source/golang/client/src/martilq/utility.go b/source/golang/client/src/martilq/utility.go index 253206d..5ac5853 100644 --- a/source/golang/client/src/martilq/utility.go +++ b/source/golang/client/src/martilq/utility.go @@ -1,2 +1,93 @@ package martilq +import ( + "os" + "path" + "time" + "strings" + "log" + "errors" +) + +func GetLogName() string { + + date := time.Now().Format("2006-01-02") + logPathName := "./logs" + + if (logPathName == "") { + return "./logs" + } + + if _, err := os.Stat(logPathName); errors.Is(err, os.ErrNotExist) { + err := os.Mkdir(logPathName, os.ModePerm) + if err != nil { + log.Println(err) + } + } + + logName := GetSoftwareName() + "_" + date + ".log" + sFullPath := path.Join(logPathName, logName ) + + _, err := os.Stat(sFullPath) + if err != nil && os.IsNotExist(err) { + log.Println("Log path: "+ sFullPath) + } + + return sFullPath +} + + +func WriteLog(LogEntry string) { + + sFullPath := GetLogName() + if (sFullPath != "") { + logFile, err := os.OpenFile(sFullPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + log.Fatal(err) + } + defer logFile.Close() + log.SetOutput(logFile) + } + log.Println(LogEntry) +} + +func OpenLog() { + dateTime := time.Now().Format("2006-01-02T15:04:05-0700") + WriteLog( "***********************************************************************************") + WriteLog( "* Start of processing: ["+dateTime+"]") + WriteLog( "***********************************************************************************") +} + +func CloseLog() { + dateTime := time.Now().Format("2006-01-02T15:04:05-0700") + WriteLog( "***********************************************************************************") + WriteLog( "* End of processing: ["+dateTime+"]") + WriteLog( "***********************************************************************************") +} + + +func NewLocalTempFile(UrlPath string, configuration *Configuration, TempPath string) string { + + parts := strings.Split(UrlPath, "/") + doc_name := parts[len(parts)-1] + temp_dir := TempPath + + if (temp_dir == "") { + if (configuration == nil) { + temp_dir = NewConfiguration().tempPath + } else { + temp_dir = configuration.tempPath + } + } + + if _, err := os.Stat(temp_dir); errors.Is(err, os.ErrNotExist) { + err := os.Mkdir(temp_dir, os.ModePerm) + if err != nil { + log.Println(err) + } else { + WriteLog("Created temp folder : " + temp_dir) + } + } + + return path.Join(temp_dir, doc_name ) +} diff --git a/source/golang/client/src/tool.go b/source/golang/client/src/tool.go index 0777273..c0c2940 100644 --- a/source/golang/client/src/tool.go +++ b/source/golang/client/src/tool.go @@ -142,13 +142,13 @@ func loadArguments(args []string) { params.description = args[ix] } } else { - panic("Missing parameter for DECRIPTION") + panic("Missing parameter for DESCRIPTION") } } if !matched && args[ix] != "--" { - fmt.Println("Unrecognised command line argument: " + args[ix]) + fmt.Println("Unrecognized command line argument: " + args[ix]) } ix = ix + 1 @@ -160,7 +160,7 @@ func printHelp() { fmt.Println("") fmt.Println("\t martilqcli_client ") - fmt.Println("\t =======++======== ") + fmt.Println("\t ================= ") fmt.Println("") fmt.Println("\tThis program is intended as a simple reference implementation") fmt.Println("\tin Go of the MartiLQ framework. It is does not provide all") @@ -171,10 +171,10 @@ func printHelp() { fmt.Println("") fmt.Println(" -h or --help : Display this help") fmt.Println(" -t or --task : Execute a predefined task which are") - fmt.Println(" INIT initialise a new configuration file") + fmt.Println(" INIT initialize a new configuration file") fmt.Println(" MAKE make a MartiLQ definition file") fmt.Println(" GET resources based on MartiLQ definition file") - fmt.Println(" RECON reconicile a MartiLQ definition file") + fmt.Println(" RECON reconcile a MartiLQ definition file") fmt.Println(" -c or --config : Configuration file used by all tasks") fmt.Println(" This is the file written by the INIT task") fmt.Println(" -s or --source : Source directory or file to build MartiLQ definition") @@ -218,62 +218,13 @@ func main () { } else { - if params.task == "INIT" { - if params.configPath == "" { - panic("Missing 'config' parameter") - } - - _, err := os.Stat(params.configPath) - if err == nil { - panic("MartiLQ configuration file '"+ params.configPath+"' already exists") - } - - c := martilq.NewConfiguration() - if c.SaveConfig(params.configPath) != true { - panic("Configuration not saved to: "+ params.configPath) - } - fmt.Println("Created MARTILQ INI definition: " + params.configPath) - matched = true - } - - if params.task == "MAKE" { - - if params.sourcePath == "" { - panic("Missing 'source' parameter") - } - if params.definitionPath == "" { - panic("Missing 'output' parameter") - } - - _, err := os.Stat(params.definitionPath) - if err == nil && params.update == false { - panic("MartiLQ document '"+ params.definitionPath+"' already exists and update not specified") - } - - m := martilq.Make(params.configPath, params.sourcePath, params.filter, params.recursive, params.urlPrefix, params.definitionPath ) - if params.title != "" { - m.Title = params.title - } - if params.description != "" { - m.Description = params.description - } - m.Save(params.definitionPath) - - fmt.Println("Created MARTILQ definition: " + params.definitionPath) - matched = true - } - - if params.task == "GET" { - fmt.Println("ET task not implemented") - matched = true - } - - if params.task == "RECON" { - _ = martilq.ReconcileFilePath(params.configPath, params.sourcePath, params.recursive, params.definitionPath, params.outputPath ) - + m = martilq.ReconcileFilePath(params.configPath, params.sourcePath, params.recursive, params.definitionPath, params.outputPath ) + matched = true + + } if !matched { diff --git a/source/golang/server/src/docs/acknowledgement.md b/source/golang/server/src/docs/acknowledgement.md new file mode 100644 index 0000000..94b9b04 --- /dev/null +++ b/source/golang/server/src/docs/acknowledgement.md @@ -0,0 +1,68 @@ +# Acknowledgment + +Once the **martiLQ** document is received by a consumer then communicating the receipt, processing, +success or failure completes the feedback loop and builds an extra layer of assurance for the organization. + +The acknowledgement workflow provides the necessary feedback. If an acknowledgement is required as part of the +consumption design then the following is approach is recommended. + +1. The publisher provides callback details. For extra security the callback details should be signed. +2. The consumer will acknowledge the receipt of the **martiLQ** document by sending back the same + document to the publisher with some values changed. +3. Change the root consumer and state (not resource) from ``active`` to ``receipt`` and modify the ``stateModified`` timestamp. +4. Change the ``consumer`` data value to only be your identifier and not others, so that the publisher + can identify the consumer and associate it with success or failure. This change to consumer value + applies to all subsequent acknowledgement messages. +5. Send the changed **martiLQ** document back using the callback details +6. On fetching each resource the resource state is changed from ``active`` to ``received`` and modify the ``stateModified`` timestamp. + If any resource cannot be retrieved the state is changed from ``active`` to ``missing`` and ``stateModified`` timestamp is udpated. +7. The consumer can elect to send back the **martiLQ** document to the publisher on each fetch or at the completion + of all fetches. The recommendation is to send at the end of all fetches because if there are issues then + having all the failures for analysis should assist in determining the extent of the failure. +8. Once all resources are fetched (or failed), the root state is changed from ``receipt`` to ``received`` and update the ``stateModified`` + timestamp if no errors occurred in retrieving the resources. If a single or many errors occurred, then the root state is + changed from ``receipt`` to ``missing`` and the ``stateModified`` timestamp is updated. The updated document is sent back + to the publisher using the callback details. +9. The next stage is to validate and process the resources defined in the **martiLQ** document. This follows + a similar process to fetching the resources. +10. On processing each resource the resource state is changed from ``received`` to ``processed`` and modify the ``stateModified`` timestamp. + If any resource cannot be processed the state is changed from ``received`` to ``error`` and update the ``stateModified`` timestamp. Once + again this can be acknowledged back to the publisher. +11. Once all resources are processed (or failed), the root state is changed from ``received`` to ``processed`` and the ``stateModified`` timestamp + updated if no errors occurred in processing the resources. If a single or many errors occurred, then the root state is + changed from ``received`` to ``error`` and the ``stateModified`` timestamp modified. The updated document is sent back to the publisher using + the callback details. + +This completes the acknowledgment workflow for the **martiLQ** document. The level of acknowledgement feedback +you wish to implement as a consumer is your decision. Any publisher providing callback details for acknowledgement can also +choose their behavior on actions and recording any acknowledgments received. + +In the above acknowledgement process, you **must not** change the identifiers in the **martiLQ** document and you **should not** +change other data except the ``consumer`` and ``state`` and ``stateModified``. + +If you are the publisher and expect acknowledgment then there is an extra scenario you need to cater for. The scenario is +that you do not receive any acknowledgement back from the expected consumer(s) within the agreed time frame. In this situation +the publisher will need to know each consumer and their service level agreements. + +## Callback + +The callback method can take any form. The normal expectation is that the same method used to communicate the **martiLQ** +document is used for the callback for example: + +* If the **martiLQ** document is originally sent by Kafka queue then the callback should use a Kafka queue separate topic +* If the **martiLQ** document is originally sent by REST API then the callback is a publisher REST API end point +* If the **martiLQ** document is originally sent by email then the callback uses a reply address for the callback + +While the above is the expected convention, you can mix and match. For example: + +* If the **martiLQ** document is originally sent by REST API then the callback can be an email address. This + situation could be acceptable if the publisher does not have the ability to accept REST API call backs. + +## Compressed file handling + +When the **martiLQ** document is defining a parent compressed file, e.g. ZIP or 7Z, then the resources are expected +to be in the compressed file. These resources can still be checked for existence and that they can be extracted. The +state of the resource is still changed to reflect the processing. + +If the file cannot be extracted either because it has not been included or there is a decompression error, then the +same acknowledgement process of using the state is used. diff --git a/source/golang/server/src/docs/comparison.md b/source/golang/server/src/docs/comparison.md index da9a3bc..2feee87 100644 --- a/source/golang/server/src/docs/comparison.md +++ b/source/golang/server/src/docs/comparison.md @@ -1,11 +1,10 @@ -Comparison of martiLQ document definition -========================================= +# Comparison of martiLQ document definition The use of metadata definitions is not unique and examples exist in many different situations. Some are standard and open while others are closed. -Some open standards are EXIF data for pictures, SQL DDL defintions +Some open standards are EXIF data for pictures, SQL DDL definitions for databases, the XMP definition and web header responses before the web content. @@ -23,18 +22,16 @@ a CKAN source into a **martiLQ** document definition and then process the data through the pipeline as you would for any other data file that had a **martiLQ** document definition. -Benefit of CKAN and martiLQ ---------------------------- +## Benefit of CKAN and martiLQ The CKAN is excellent at defining the data source details but it lacks information -for load quality. If you have CKAN deployed in your organisation and wish +for load quality. If you have CKAN deployed in your organization and wish exhange or process the data referenced in CKAN, then there are synergies between CKAN and marti. Samples exist on CKAN integration. -Magda and martiLQ ------------------ +## Magda and martiLQ Another source of data is [Magda](https://magda.io/) which has API metadata definitions. Magda is more about data federation and as such provides diff --git a/source/golang/server/src/docs/custom.md b/source/golang/server/src/docs/custom.md index 2ce9685..4f53aff 100644 --- a/source/golang/server/src/docs/custom.md +++ b/source/golang/server/src/docs/custom.md @@ -1,7 +1,7 @@ Custom Definition ================= -The custome definition section allows the inclusion of extensions +The custom definition section allows the inclusion of extensions to the standard. To demonstrate the inclusion, there are three sample extensions. These are: diff --git a/source/golang/server/src/docs/filetransfer_ack.png b/source/golang/server/src/docs/filetransfer_ack.png new file mode 100644 index 0000000..600936f Binary files /dev/null and b/source/golang/server/src/docs/filetransfer_ack.png differ diff --git a/source/golang/server/src/docs/filetransfer_appcode.png b/source/golang/server/src/docs/filetransfer_appcode.png new file mode 100644 index 0000000..db5446d Binary files /dev/null and b/source/golang/server/src/docs/filetransfer_appcode.png differ diff --git a/source/golang/server/src/docs/filetransfer_base.png b/source/golang/server/src/docs/filetransfer_base.png new file mode 100644 index 0000000..a69f707 Binary files /dev/null and b/source/golang/server/src/docs/filetransfer_base.png differ diff --git a/source/golang/server/src/docs/filetransfer_injection.png b/source/golang/server/src/docs/filetransfer_injection.png new file mode 100644 index 0000000..c54c727 Binary files /dev/null and b/source/golang/server/src/docs/filetransfer_injection.png differ diff --git a/source/golang/server/src/docs/magda.md b/source/golang/server/src/docs/magda.md index 27e76fd..ba0b60d 100644 --- a/source/golang/server/src/docs/magda.md +++ b/source/golang/server/src/docs/magda.md @@ -1,5 +1,5 @@ -Magda definitions -================= +# Magda definitions + https://magda.io/ diff --git a/source/golang/server/src/docs/martiLQ.md b/source/golang/server/src/docs/martiLQ.md index 4526dc1..2f3f113 100644 --- a/source/golang/server/src/docs/martiLQ.md +++ b/source/golang/server/src/docs/martiLQ.md @@ -39,7 +39,7 @@ modified|Modified date and time of the **martiLQ** document|Now tags|List of tags or keywords| publisher|Publisher name| contactPoint|Contact point of a person or team| -accessLevel|Acces level| +accessLevel|Access level| rights|Rights| * Batch @@ -54,7 +54,7 @@ rights|Rights| ### Information extension The information supplied can be extended by party agreement and there -are place holders in the defintion. +are place holders in the definition. ## Resource @@ -62,7 +62,7 @@ The resource section is a list of documents or files that are to be grouped together are listed under the same **martiLQ** definition. At least one document or file must be included. If the same resource is repeated -it will commonly be for definiting multiple formats, with each file having a +it will commonly be defining multiple formats, with each file having a different extension. Commonly the definition includes at least the following items: @@ -84,7 +84,7 @@ for more details Name|Description|Default or values ---|---|-- hash|Hash of document - The hash of the document, which can be blank especially for large documents -algo|Hash algorithm - Algoroithm used to generate the hash value or sign it +algo|Hash algorithm - Algorithm used to generate the hash value or sign it description|Description - A more detailed description version|Version - A document version encoding|Encoding @@ -117,6 +117,7 @@ sample can be generated using the GOLANG client program with parameters: "tags": null, "license": "", "state": "active", + "stateModified": "2021-11-02T22:44:29.6887001+11:00", "batch": 1.001, "describedBy": "", "landingPage": "", @@ -130,6 +131,7 @@ sample can be generated using the GOLANG client program with parameters: "modified": "2021-11-02T07:47:13.9410018+11:00", "expires": "2023-11-02T00:00:00+11:00", "state": "active", + "stateModified": "2021-11-02T22:44:29.6881663+11:00", "author": "", "length": 3654, "hash": { diff --git a/source/golang/server/src/docs/quality.md b/source/golang/server/src/docs/quality.md index 8b84489..f9658f0 100644 --- a/source/golang/server/src/docs/quality.md +++ b/source/golang/server/src/docs/quality.md @@ -19,7 +19,7 @@ load quality metrics. * Number of records in the document - This is the number of data primary records not the count of end of lines and is agreed between parties. XML record counts could be based on the number of primary segments under root. JSON records can be counted in a similar way. - The headers or trailling records are not counted + The headers or trailing records are not counted ## Addresses deficiencies diff --git a/source/golang/server/src/docs/references.md b/source/golang/server/src/docs/references.md index 9e0d285..379a351 100644 --- a/source/golang/server/src/docs/references.md +++ b/source/golang/server/src/docs/references.md @@ -1,7 +1,7 @@ # References The following are references to documents that inspired the creation of **martiLQ** -document and associatd framework. +document and associated framework. https://dex.dss.gov.au/sites/default/files/documents/2021-06/data-exchange-protocols-june-2021.pdf diff --git a/source/golang/server/src/docs/what.md b/source/golang/server/src/docs/what.md index 199d925..0d551c8 100644 --- a/source/golang/server/src/docs/what.md +++ b/source/golang/server/src/docs/what.md @@ -12,7 +12,7 @@ for various programming languages and situations. As many programming languages generate portable programs that can execute on multiple operating systems, the likelihood is that a tools exists for you. -The source for tools is provided in the Github repository and some have precompiled +The source for tools is provided in the Github repository and some have pre-compiled images. See the project source directory for more details. diff --git a/source/golang/server/src/docs/who.md b/source/golang/server/src/docs/who.md index cb8bf4c..afea3b6 100644 --- a/source/golang/server/src/docs/who.md +++ b/source/golang/server/src/docs/who.md @@ -1,13 +1,11 @@ -Who is likely to use martiLQ -============================ +# Who is likely to use martiLQ You are likely to find the **martiLQ** framework relevant if you: 1. Have many document exchanges, such as End of Day batches 2. Need to verify or reconcile the documents -Data exchanges --------------- +## Data exchanges If you are creating or receiving many documents or files on a regular basis then you probably have some framework defined. The framework may be as simple as: @@ -24,8 +22,7 @@ Simple framework such as the above have limitations, such as: * Lower automation prospects and alignment to DataSecOps * Poor fit to web applications (they tend to be designed for FTP and LAN) -Framework Sidecar files ------------------------ +## Framework Sidecar files The **martiLQ** framework addresses the issues and limitations by using sidecar or shadow files. The [concept of sidecar files](https://en.wikipedia.org/wiki/Sidecar_file) is @@ -35,7 +32,7 @@ Sidecar files can also be implemented as ``forks`` and built into the operating in Mac OS X HFS. The Microsoft NTFS supports Alternate Data Streams to achieve a similar outcome. Unfortunately this information is not transferrable to other systems. -The proposition is to define a format for the sidecare file and provide common library tools that +The proposition is to define a format for the sidecar file and provide common library tools that can be be used on multiple platforms when exchanging documents / files. Multiple documents can be -defined in a singel **martiLQ** definition which adds to efficiency and productivity if used +defined in a single **martiLQ** definition which adds to efficiency and productivity if used for End of Day or similar batches - or even single file transfers. diff --git a/source/martilq.ini b/source/martilq.ini index 66291e8..b7f2e84 100644 --- a/source/martilq.ini +++ b/source/martilq.ini @@ -37,3 +37,10 @@ signKey_Password = proxy = username = password = + +[Smtp] + +host = +port = +username = +password = diff --git a/source/powershell/MartiLQ.ps1 b/source/powershell/MartiLQ.ps1 index dc5dfb2..45ee9d4 100644 --- a/source/powershell/MartiLQ.ps1 +++ b/source/powershell/MartiLQ.ps1 @@ -5,7 +5,7 @@ . .\source\powershell\MartiLQAttribute.ps1 -function New-MartiDefinition +function New-MartiLQDefinition { Param( [String] $ConfigPath = $null @@ -23,13 +23,6 @@ function New-MartiDefinition renderer = "MARTILQREFERENCE:Mustache" url = "" } - - $oAcknowledgement = [PSCustomObject]@{ - url = "" - algo = "" - value = "" - signed = $false - } if ($null -eq $ConfigPath -or $ConfigPath -eq "") { @@ -97,7 +90,7 @@ function New-MartiDefinition publisher = $publisher contactPoint = $oConfig.contactPoint accessLevel = $oConfig.accessLevel - consumer = $lconsumer + consumers = $lconsumer rights = $oConfig.rights license = $oConfig.license state = $oConfig.state @@ -108,7 +101,7 @@ function New-MartiDefinition theme =$oConfig.theme resources = $lresource - acknowledge = $oAcknowledgement + acknowledge = Get-Acknowledgement custom = $lCustom } @@ -116,6 +109,94 @@ function New-MartiDefinition } +function Save-MartiLQDefinition +{ + Param( + [Parameter(Mandatory)][PSCustomObject] $MartiLQ, + [Parameter(Mandatory)][String] $FilePath + ) + + $fileJson = $FilePath + $MartiLQ | ConvertTo-Json -depth 100 | Out-File $fileJson + + return $fileJson +} + + +function Restore-MartiLQDefinition +{ + Param( + [Parameter(Mandatory)][String] $FilePath + ) + + $oMartiLQ = [PSCustomObject](Get-Content -Raw $FilePath | Out-String | ConvertFrom-Json) + + $version = Get-DefinitionVersion -MartiLQ $oMartiLQ + if ($version -lt "0.0.2") { + if (![bool]($oMartiLQ.PSCustomObject.Properties.name -match "consumers")) { + [System.Collections.ArrayList]$lconsumer = @() + $oMartiLQ | Add-Member -Name "consumers" -Type NoteProperty -Value $lconsumer + } + + if (![bool]($oMartiLQ.PSCustomobject.Properties.name -match "acknowledge")) { + $oMartiLQ | Add-Member -Name "acknowledge" -Type NoteProperty -Value (Get-Acknowledgement) + } + + Set-DefinitionVersion -MartiLQ $oMartiLQ -Version "0.0.2" + $newVersion = Get-DefinitionVersion -MartiLQ $oMartiLQ + Write-Host "Updating from version $version to $newVersion" + } + + return $oMartiLQ +} + +function Get-DefinitionVersion +{ +param ( + [Parameter(Mandatory)][PSCustomObject] $MartiLQ +) + + $version = "0.0.1" + $MartiLQ.custom | ForEach-Object { + if ($_.extension -eq "software" -and $_.softwareName -eq (Get-SoftwareName)) { + $version = $_.version + } + } + + return $version +} + +function Set-DefinitionVersion +{ +param ( + [Parameter(Mandatory)][PSCustomObject] $MartiLQ, + [Parameter(Mandatory)][String] $Version +) + + $MartiLQ.custom | ForEach-Object { + + if ($_.extension -eq "software" -and $_.softwareName -eq (Get-SoftwareName)) { + $_.version = $Version + #return $MartiLQ + } + + } + + #return $MartiLQ +} + +function Get-Acknowledgement{ + + $oAcknowledgement = [PSCustomObject]@{ + url = "" + algo = "" + value = "" + signed = $false + } + + return $oAcknowledgement +} + function Set-MartiAttribute { Param( @@ -244,9 +325,6 @@ function Get-MartiResource { } - - - function ConvertFrom-Ckan { Param( @@ -341,7 +419,6 @@ Param( } - function Compress-MartiLQ { Param( @@ -397,6 +474,3 @@ Param( Close-Log } - - - diff --git a/source/powershell/MartiLQConfiguration.ps1 b/source/powershell/MartiLQConfiguration.ps1 index 3efaa9e..67aaa5e 100644 --- a/source/powershell/MartiLQConfiguration.ps1 +++ b/source/powershell/MartiLQConfiguration.ps1 @@ -1,7 +1,7 @@ -$script:SoftwareVersion = "0.0.1" +$script:SoftwareVersion = "0.0.2" $global:default_metaFile = "##martilq##.json" $global:martiLQ_configuration = $null @@ -58,6 +58,11 @@ function Get-DefaultConfiguration { proxy_User = $null proxy_Credential = $null + smtp_Host = $null + smtp_Port = $null + smtp_User = $null + smtp_Credential = $null + loaded = $false } @@ -155,6 +160,13 @@ function Import-Configuration { $oConfig.hashAlgorithm = Set-ConfigurationValue $oConfig.hashAlgorithm -Value $iConfig.Hash.hashAlgorithm } + if ($null -ne $iConfig.Smtp) { + $oConfig.smtp_Host = Set-ConfigurationValue $oConfig.smtp_Host -Value $iConfig.Smtp.host + $oConfig.smtp_Port = Set-ConfigurationValue $oConfig.smtp_Port -Value $iConfig.Smtp.port + $oConfig.smtp_User = Set-ConfigurationValue $oConfig.smtp_Port -Value $iConfig.Smtp.username + $oConfig.smtp_Credential = Set-ConfigurationValue $oConfig.smtp_Credential -Value $iConfig.Smtp.password + } + } $global:martiLQ_configuration = $oConfig diff --git a/source/powershell/MartiLQTool.ps1 b/source/powershell/MartiLQTool.ps1 new file mode 100644 index 0000000..98fb7cc --- /dev/null +++ b/source/powershell/MartiLQTool.ps1 @@ -0,0 +1,186 @@ + +. .\source\powershell\MartiLQ.ps1 +. .\source\powershell\MartiLQConfiguration.ps1 + + +function Send-EmailAck { + param ( + [String] $FileAttachment, + [String] $Recipient, + [String] $State, + [int] $Buffersize = 1024 + ) + + $receiver = $Recipient.Substring(5) + Write-Host "Sending acknowledgment via email to: $receiver " -ForegroundColor Green + + $EmailFrom = $env:MARTILQ_EMAIL_FROM + + $Subject = "martiLQ acknowledge [$State]" + $Body = "Simple email ack" + + $password = ConvertTo-SecureString $env:MARTILQ_EMAIL_PASSWORD -AsPlainText -Force + $credential = New-Object System.Management.Automation.PSCredential ($env:MARTILQ_EMAIL_USERNAME, $password) + + Write-Host "SMTP: $($env:MARTILQ_EMAIL_HOST) :: 465 " -ForegroundColor Yellow + Write-Host "Send with: $FileAttachment :: $Subject :: $Body " -ForegroundColor Yellow + + $att = new-object Net.Mail.Attachment($FileAttachment) + + #Send mail with attachment + $from = New-Object System.Net.Mail.MailAddress($EmailFrom) + $to = New-Object System.Net.Mail.MailAddress($receiver) + $email = New-Object System.Net.Mail.Mailmessage($from, $to) + $email.Subject = $Subject + $email.Body = $Body + $email.IsBodyHTML = $true + $email.Attachments.Add($att) + + $att.Dispose() + + $smtp = New-Object Net.Mail.SmtpClient($env:MARTILQ_EMAIL_HOST, 465) + $smtp.EnableSSL = $true + $smtp.Credentials = New-Object System.Net.NetworkCredential($env:MARTILQ_EMAIL_USERNAME, $env:MARTILQ_EMAIL_PASSWORD) + $smtp.Send($email) + +} + +function Get-Resources { + param ( + [Parameter(Mandatory)][PSCustomObject] $MartiLQ, + [String] $DataPath, + [String] $CurrentState, + [String] $Consumer + ) + + $nextState = "received" + + Try { + foreach ($item in $MartiLQ.resources) { + if ($item.state -eq $currentState) { + + $item.state = $nextState + } + } + + if ($Consumer -ne "") { + [System.Collections.ArrayList]$lconsumer = @() + $lconsumer += $Consumer + $MartiLQ.consumers = $lconsumer + } + + $today = Get-Date + $dateToday = $today.Tostring("yyyy-MM-ddTHH:mm:ss") + $MartiLQ.stateModified = $dateToday + $MartiLQ.state = $nextState + + # Notification + $ack = $MartiLQ.acknowledge + + if ($ack.url.startswith("mail:")) + { + $fileJson = "./temp/MartiBSBRemote_interim2.json" + $attachment = Save-MartiLQDefinition -MartiLQ $oMarti -FilePath $fileJson + Send-EmailAck -FileAttachment $attachment -Recipient $ack.url -State $nextState + } + + } Catch { + Write-Host "Error in resource get: $_" + } + + return $MartiLQ +} + + +function Test-Resource { + param ( + [Parameter(Mandatory)][PSCustomObject] $MartiLQ, + [String] $DataPath, + [String] $CurrentState, + [String] $Consumer + ) + + $nextState = "verified" + + Try { + foreach ($item in $MartiLQ.resources) { + if ($item.state -eq $currentState) { + + $item.state = $nextState + } + } + + if ($Consumer -ne "") { + [System.Collections.ArrayList]$lconsumer = @() + $lconsumer += $Consumer + $MartiLQ.consumers = $lconsumer + } + + $MartiLQ.state = $nextState + } Catch { + Write-Host "Error in resource test: $_" + } + + return $MartiLQ +} + +function Invoke-ProcessResource { + param ( + [Parameter(Mandatory)][PSCustomObject] $MartiLQ, + [String] $DataPath, + [String] $CurrentState, + [String] $Consumer + ) + + $nextState = "processed" + + Try { + foreach ($item in $MartiLQ.resources) { + if ($item.state -eq $currentState) { + + $item.state = $nextState + } + } + + if ($Consumer -ne "") { + [System.Collections.ArrayList]$lconsumer = @() + $lconsumer += $Consumer + $MartiLQ.consumers = $lconsumer + } + + $MartiLQ.state = $nextState + } Catch { + Write-Host "Error in resource process: $_" + } + + return $MartiLQ +} + +$currentState = "expired" +$nextState = "active" +$consumer = "Test-Framework" + +$fileJson = "C:\Users\meerkat\source\marti\docs\source\samples\powershell\test\MartiBSBRemote.json" +$oMarti = Restore-MartiLQDefinition -FilePath $fileJson + +$oMarti.acknowledge.url = "mail:tp_reklam@villacentrum.com" + +$today = Get-Date +$dateToday = $today.Tostring("yyyy-MM-ddTHH:mm:ss") +$oMarti.stateModified = $dateToday +$oMarti.state = $nextState + +foreach ($item in $oMarti.resources) { + if ($item.state -eq $currentState) { + $item.state = $nextState + } +} + +$oMarti = Get-Resources -MartiLQ $oMarti[0] -DataPath "" -CurrentState "active" -Consumer $consumer + +$oMarti = Test-Resource -MartiLQ $oMarti -DataPath "" -CurrentState "received" -Consumer $consumer + +$oMarti = Invoke-ProcessResource -MartiLQ $oMarti -DataPath "" -CurrentState "verified" -Consumer $consumer + +$fileJson = "C:\Users\meerkat\source\marti\docs\source\samples\powershell\test\MartiBSBRemote_v2.json" +Save-MartiLQDefinition -MartiLQ $oMarti -FilePath $fileJson diff --git a/source/powershell/MartiLQUtilities.ps1 b/source/powershell/MartiLQUtilities.ps1 index 6900a0e..4a59e19 100644 --- a/source/powershell/MartiLQUtilities.ps1 +++ b/source/powershell/MartiLQUtilities.ps1 @@ -91,4 +91,4 @@ function New-LocalTempFile{ } return Join-Path -Path $temp_dir -ChildPath $doc_name -} \ No newline at end of file +} diff --git a/source/python/client/martiLQ.py b/source/python/client/martiLQ.py index d16bb65..98918a2 100644 --- a/source/python/client/martiLQ.py +++ b/source/python/client/martiLQ.py @@ -24,14 +24,13 @@ from mutility import mUtility class martiLQ: - _SoftwareVersion = "0.0.1" _default_metaFile = "##martilq##.json" _oSoftware = { "extension": "software", - "softwareName": "MARTILQREFERENCE", + "softwareName": mConfiguration.GetSoftwareName(), "author": "Meerkat@merebox.com", - "version": "0.0.1" + "version": mConfiguration.GetSoftwareVersion() } _oTemplate = { diff --git a/source/python/client/mconfiguration.py b/source/python/client/mconfiguration.py index d0a4a9c..c999259 100644 --- a/source/python/client/mconfiguration.py +++ b/source/python/client/mconfiguration.py @@ -19,24 +19,26 @@ from mlogging import mLogging class mConfiguration: - _SoftwareVersion = "0.0.1" + _SoftwareVersion = "0.0.2" _default_metaFile = "##martilq##.json" _oSoftware = { "extension": "software", - "softwareName": "MARTILQREFERENCE", + "softwareName": GetSoftwareName(), "author": "Meerkat@merebox.com", - "version": "0.0.1" + "version": GetSoftwareVersion() } - _oConfiguration = None _Log = None - def GetSoftwareName(self): + def GetSoftwareName(): return "MARTILQREFERENCE" + def GetSoftwareVersion(): + return _SoftwareVersion + def __init__(self): self._oConfiguration = {