updates to Go and Python functions
parent
a54c3e000c
commit
b96b311db7
|
|
@ -12,12 +12,12 @@ import (
|
|||
|
||||
const cSoftwareName = "MARTILQREFERENCE"
|
||||
const cSoftwareAuthor = "Meerkat@merebox.com"
|
||||
const cSoftwareVersion = "0.0.1"
|
||||
const cSoftwareVersion = "0.0.2"
|
||||
const cIniFileName = "martilq.ini"
|
||||
const cExpires = "t:7:0:0"
|
||||
const cEncoding = ""
|
||||
|
||||
type configuration struct {
|
||||
type Configuration struct {
|
||||
softwareName string
|
||||
|
||||
dateFormat string
|
||||
|
|
@ -68,14 +68,16 @@ func GetSoftwareName() string {
|
|||
return cSoftwareName
|
||||
}
|
||||
|
||||
func NewConfiguration() configuration {
|
||||
func NewConfiguration() Configuration {
|
||||
|
||||
c := configuration {}
|
||||
c := Configuration {}
|
||||
|
||||
c.softwareName = GetSoftwareName()
|
||||
|
||||
c.dateFormat = "2006-01-02"
|
||||
c.dateTimeFormat = "2006-01-02T15:04:05-0700"
|
||||
c.dataPath = ""
|
||||
c.tempPath = "temp"
|
||||
|
||||
c.title = "{{documentName}}"
|
||||
c.state = "active"
|
||||
|
|
@ -95,6 +97,8 @@ func NewConfiguration() configuration {
|
|||
c.spatial = GetSpatial()
|
||||
c.temporal = GetTemporal()
|
||||
|
||||
|
||||
|
||||
configPath := findIni()
|
||||
if configPath != "" {
|
||||
c.LoadConfig(configPath)
|
||||
|
|
@ -145,7 +149,7 @@ func findIni() string {
|
|||
return foundPath
|
||||
}
|
||||
|
||||
func (c *configuration) SaveConfig(ConfigPath string) bool {
|
||||
func (c *Configuration) SaveConfig(ConfigPath string) bool {
|
||||
|
||||
cfgini, _ := ini.LooseLoad("./martilq.ini")
|
||||
|
||||
|
|
@ -197,7 +201,7 @@ func (c *configuration) SaveConfig(ConfigPath string) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
func (c *configuration) LoadConfig(ConfigPath string) bool {
|
||||
func (c *Configuration) LoadConfig(ConfigPath string) bool {
|
||||
|
||||
if ConfigPath != "" {
|
||||
_, err := os.Stat(ConfigPath)
|
||||
|
|
@ -286,7 +290,7 @@ func Loadenv(key string, default_value string ) string {
|
|||
}
|
||||
|
||||
|
||||
func (c *configuration) ExpireDate(sourcePath string ) time.Time {
|
||||
func (c *Configuration) ExpireDate(sourcePath string ) time.Time {
|
||||
|
||||
var expires time.Time
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +0,0 @@
|
|||
package martilq
|
||||
|
||||
|
||||
import (
|
||||
"log"
|
||||
)
|
||||
|
||||
func WriteLog(entry string) {
|
||||
log.Printf(entry)
|
||||
}
|
||||
|
|
@ -44,7 +44,7 @@ type Marti struct {
|
|||
Custom []interface{} `json:"custom"`
|
||||
|
||||
|
||||
config configuration
|
||||
config Configuration
|
||||
}
|
||||
|
||||
func NewMarti() Marti {
|
||||
|
|
@ -63,7 +63,7 @@ func NewMarti() Marti {
|
|||
return m
|
||||
}
|
||||
|
||||
func Load(c configuration, pathFile string) (Marti, error) {
|
||||
func Load(c Configuration, pathFile string) (Marti, error) {
|
||||
|
||||
m := Marti {}
|
||||
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ type Resource struct {
|
|||
Attributes []Attribute `json:"attributes"`
|
||||
}
|
||||
|
||||
func NewResource(config configuration) Resource {
|
||||
func NewResource(config Configuration) Resource {
|
||||
|
||||
r := Resource {}
|
||||
u := uuid.New()
|
||||
|
|
@ -57,7 +57,7 @@ func NewResource(config configuration) Resource {
|
|||
return r
|
||||
}
|
||||
|
||||
func NewMartiLQResource(config configuration, sourcePath string, urlPath string, excludeHash bool, extendAttributes bool) (Resource, error) {
|
||||
func NewMartiLQResource(config Configuration, sourcePath string, urlPath string, excludeHash bool, extendAttributes bool) (Resource, error) {
|
||||
|
||||
r := Resource {}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,2 +1,93 @@
|
|||
package martilq
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
"strings"
|
||||
"log"
|
||||
"errors"
|
||||
)
|
||||
|
||||
func GetLogName() string {
|
||||
|
||||
date := time.Now().Format("2006-01-02")
|
||||
logPathName := "./logs"
|
||||
|
||||
if (logPathName == "") {
|
||||
return "./logs"
|
||||
}
|
||||
|
||||
if _, err := os.Stat(logPathName); errors.Is(err, os.ErrNotExist) {
|
||||
err := os.Mkdir(logPathName, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
logName := GetSoftwareName() + "_" + date + ".log"
|
||||
sFullPath := path.Join(logPathName, logName )
|
||||
|
||||
_, err := os.Stat(sFullPath)
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
log.Println("Log path: "+ sFullPath)
|
||||
}
|
||||
|
||||
return sFullPath
|
||||
}
|
||||
|
||||
|
||||
func WriteLog(LogEntry string) {
|
||||
|
||||
sFullPath := GetLogName()
|
||||
if (sFullPath != "") {
|
||||
logFile, err := os.OpenFile(sFullPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer logFile.Close()
|
||||
log.SetOutput(logFile)
|
||||
}
|
||||
log.Println(LogEntry)
|
||||
}
|
||||
|
||||
func OpenLog() {
|
||||
dateTime := time.Now().Format("2006-01-02T15:04:05-0700")
|
||||
WriteLog( "***********************************************************************************")
|
||||
WriteLog( "* Start of processing: ["+dateTime+"]")
|
||||
WriteLog( "***********************************************************************************")
|
||||
}
|
||||
|
||||
func CloseLog() {
|
||||
dateTime := time.Now().Format("2006-01-02T15:04:05-0700")
|
||||
WriteLog( "***********************************************************************************")
|
||||
WriteLog( "* End of processing: ["+dateTime+"]")
|
||||
WriteLog( "***********************************************************************************")
|
||||
}
|
||||
|
||||
|
||||
func NewLocalTempFile(UrlPath string, configuration *Configuration, TempPath string) string {
|
||||
|
||||
parts := strings.Split(UrlPath, "/")
|
||||
doc_name := parts[len(parts)-1]
|
||||
temp_dir := TempPath
|
||||
|
||||
if (temp_dir == "") {
|
||||
if (configuration == nil) {
|
||||
temp_dir = NewConfiguration().tempPath
|
||||
} else {
|
||||
temp_dir = configuration.tempPath
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := os.Stat(temp_dir); errors.Is(err, os.ErrNotExist) {
|
||||
err := os.Mkdir(temp_dir, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
WriteLog("Created temp folder : " + temp_dir)
|
||||
}
|
||||
}
|
||||
|
||||
return path.Join(temp_dir, doc_name )
|
||||
}
|
||||
|
|
|
|||
|
|
@ -142,13 +142,13 @@ func loadArguments(args []string) {
|
|||
params.description = args[ix]
|
||||
}
|
||||
} else {
|
||||
panic("Missing parameter for DECRIPTION")
|
||||
panic("Missing parameter for DESCRIPTION")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if !matched && args[ix] != "--" {
|
||||
fmt.Println("Unrecognised command line argument: " + args[ix])
|
||||
fmt.Println("Unrecognized command line argument: " + args[ix])
|
||||
}
|
||||
|
||||
ix = ix + 1
|
||||
|
|
@ -160,7 +160,7 @@ func printHelp() {
|
|||
|
||||
fmt.Println("")
|
||||
fmt.Println("\t martilqcli_client ")
|
||||
fmt.Println("\t =======++======== ")
|
||||
fmt.Println("\t ================= ")
|
||||
fmt.Println("")
|
||||
fmt.Println("\tThis program is intended as a simple reference implementation")
|
||||
fmt.Println("\tin Go of the MartiLQ framework. It is does not provide all")
|
||||
|
|
@ -171,10 +171,10 @@ func printHelp() {
|
|||
fmt.Println("")
|
||||
fmt.Println(" -h or --help : Display this help")
|
||||
fmt.Println(" -t or --task : Execute a predefined task which are")
|
||||
fmt.Println(" INIT initialise a new configuration file")
|
||||
fmt.Println(" INIT initialize a new configuration file")
|
||||
fmt.Println(" MAKE make a MartiLQ definition file")
|
||||
fmt.Println(" GET resources based on MartiLQ definition file")
|
||||
fmt.Println(" RECON reconicile a MartiLQ definition file")
|
||||
fmt.Println(" RECON reconcile a MartiLQ definition file")
|
||||
fmt.Println(" -c or --config : Configuration file used by all tasks")
|
||||
fmt.Println(" This is the file written by the INIT task")
|
||||
fmt.Println(" -s or --source : Source directory or file to build MartiLQ definition")
|
||||
|
|
@ -218,62 +218,13 @@ func main () {
|
|||
} else {
|
||||
|
||||
|
||||
if params.task == "INIT" {
|
||||
if params.configPath == "" {
|
||||
panic("Missing 'config' parameter")
|
||||
}
|
||||
|
||||
_, err := os.Stat(params.configPath)
|
||||
if err == nil {
|
||||
panic("MartiLQ configuration file '"+ params.configPath+"' already exists")
|
||||
}
|
||||
|
||||
c := martilq.NewConfiguration()
|
||||
if c.SaveConfig(params.configPath) != true {
|
||||
panic("Configuration not saved to: "+ params.configPath)
|
||||
}
|
||||
fmt.Println("Created MARTILQ INI definition: " + params.configPath)
|
||||
matched = true
|
||||
}
|
||||
|
||||
if params.task == "MAKE" {
|
||||
|
||||
if params.sourcePath == "" {
|
||||
panic("Missing 'source' parameter")
|
||||
}
|
||||
if params.definitionPath == "" {
|
||||
panic("Missing 'output' parameter")
|
||||
}
|
||||
|
||||
_, err := os.Stat(params.definitionPath)
|
||||
if err == nil && params.update == false {
|
||||
panic("MartiLQ document '"+ params.definitionPath+"' already exists and update not specified")
|
||||
}
|
||||
|
||||
m := martilq.Make(params.configPath, params.sourcePath, params.filter, params.recursive, params.urlPrefix, params.definitionPath )
|
||||
if params.title != "" {
|
||||
m.Title = params.title
|
||||
}
|
||||
if params.description != "" {
|
||||
m.Description = params.description
|
||||
}
|
||||
m.Save(params.definitionPath)
|
||||
|
||||
fmt.Println("Created MARTILQ definition: " + params.definitionPath)
|
||||
matched = true
|
||||
}
|
||||
|
||||
if params.task == "GET" {
|
||||
fmt.Println("ET task not implemented")
|
||||
matched = true
|
||||
}
|
||||
|
||||
|
||||
if params.task == "RECON" {
|
||||
|
||||
_ = martilq.ReconcileFilePath(params.configPath, params.sourcePath, params.recursive, params.definitionPath, params.outputPath )
|
||||
|
||||
m = martilq.ReconcileFilePath(params.configPath, params.sourcePath, params.recursive, params.definitionPath, params.outputPath )
|
||||
|
||||
matched = true
|
||||
|
||||
|
||||
}
|
||||
|
||||
if !matched {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
# Acknowledgment
|
||||
|
||||
Once the **martiLQ** document is received by a consumer then communicating the receipt, processing,
|
||||
success or failure completes the feedback loop and builds an extra layer of assurance for the organization.
|
||||
|
||||
The acknowledgement workflow provides the necessary feedback. If an acknowledgement is required as part of the
|
||||
consumption design then the following is approach is recommended.
|
||||
|
||||
1. The publisher provides callback details. For extra security the callback details should be signed.
|
||||
2. The consumer will acknowledge the receipt of the **martiLQ** document by sending back the same
|
||||
document to the publisher with some values changed.
|
||||
3. Change the root consumer and state (not resource) from ``active`` to ``receipt`` and modify the ``stateModified`` timestamp.
|
||||
4. Change the ``consumer`` data value to only be your identifier and not others, so that the publisher
|
||||
can identify the consumer and associate it with success or failure. This change to consumer value
|
||||
applies to all subsequent acknowledgement messages.
|
||||
5. Send the changed **martiLQ** document back using the callback details
|
||||
6. On fetching each resource the resource state is changed from ``active`` to ``received`` and modify the ``stateModified`` timestamp.
|
||||
If any resource cannot be retrieved the state is changed from ``active`` to ``missing`` and ``stateModified`` timestamp is udpated.
|
||||
7. The consumer can elect to send back the **martiLQ** document to the publisher on each fetch or at the completion
|
||||
of all fetches. The recommendation is to send at the end of all fetches because if there are issues then
|
||||
having all the failures for analysis should assist in determining the extent of the failure.
|
||||
8. Once all resources are fetched (or failed), the root state is changed from ``receipt`` to ``received`` and update the ``stateModified``
|
||||
timestamp if no errors occurred in retrieving the resources. If a single or many errors occurred, then the root state is
|
||||
changed from ``receipt`` to ``missing`` and the ``stateModified`` timestamp is updated. The updated document is sent back
|
||||
to the publisher using the callback details.
|
||||
9. The next stage is to validate and process the resources defined in the **martiLQ** document. This follows
|
||||
a similar process to fetching the resources.
|
||||
10. On processing each resource the resource state is changed from ``received`` to ``processed`` and modify the ``stateModified`` timestamp.
|
||||
If any resource cannot be processed the state is changed from ``received`` to ``error`` and update the ``stateModified`` timestamp. Once
|
||||
again this can be acknowledged back to the publisher.
|
||||
11. Once all resources are processed (or failed), the root state is changed from ``received`` to ``processed`` and the ``stateModified`` timestamp
|
||||
updated if no errors occurred in processing the resources. If a single or many errors occurred, then the root state is
|
||||
changed from ``received`` to ``error`` and the ``stateModified`` timestamp modified. The updated document is sent back to the publisher using
|
||||
the callback details.
|
||||
|
||||
This completes the acknowledgment workflow for the **martiLQ** document. The level of acknowledgement feedback
|
||||
you wish to implement as a consumer is your decision. Any publisher providing callback details for acknowledgement can also
|
||||
choose their behavior on actions and recording any acknowledgments received.
|
||||
|
||||
In the above acknowledgement process, you **must not** change the identifiers in the **martiLQ** document and you **should not**
|
||||
change other data except the ``consumer`` and ``state`` and ``stateModified``.
|
||||
|
||||
If you are the publisher and expect acknowledgment then there is an extra scenario you need to cater for. The scenario is
|
||||
that you do not receive any acknowledgement back from the expected consumer(s) within the agreed time frame. In this situation
|
||||
the publisher will need to know each consumer and their service level agreements.
|
||||
|
||||
## Callback
|
||||
|
||||
The callback method can take any form. The normal expectation is that the same method used to communicate the **martiLQ**
|
||||
document is used for the callback for example:
|
||||
|
||||
* If the **martiLQ** document is originally sent by Kafka queue then the callback should use a Kafka queue separate topic
|
||||
* If the **martiLQ** document is originally sent by REST API then the callback is a publisher REST API end point
|
||||
* If the **martiLQ** document is originally sent by email then the callback uses a reply address for the callback
|
||||
|
||||
While the above is the expected convention, you can mix and match. For example:
|
||||
|
||||
* If the **martiLQ** document is originally sent by REST API then the callback can be an email address. This
|
||||
situation could be acceptable if the publisher does not have the ability to accept REST API call backs.
|
||||
|
||||
## Compressed file handling
|
||||
|
||||
When the **martiLQ** document is defining a parent compressed file, e.g. ZIP or 7Z, then the resources are expected
|
||||
to be in the compressed file. These resources can still be checked for existence and that they can be extracted. The
|
||||
state of the resource is still changed to reflect the processing.
|
||||
|
||||
If the file cannot be extracted either because it has not been included or there is a decompression error, then the
|
||||
same acknowledgement process of using the state is used.
|
||||
|
|
@ -1,11 +1,10 @@
|
|||
Comparison of martiLQ document definition
|
||||
=========================================
|
||||
# Comparison of martiLQ document definition
|
||||
|
||||
The use of metadata definitions is not unique and examples
|
||||
exist in many different situations. Some are standard and open
|
||||
while others are closed.
|
||||
|
||||
Some open standards are EXIF data for pictures, SQL DDL defintions
|
||||
Some open standards are EXIF data for pictures, SQL DDL definitions
|
||||
for databases, the XMP definition and web header responses before the
|
||||
web content.
|
||||
|
||||
|
|
@ -23,18 +22,16 @@ a CKAN source into a **martiLQ** document definition and then process the data
|
|||
through the pipeline as you would for any other data file that had a
|
||||
**martiLQ** document definition.
|
||||
|
||||
Benefit of CKAN and martiLQ
|
||||
---------------------------
|
||||
## Benefit of CKAN and martiLQ
|
||||
|
||||
The CKAN is excellent at defining the data source details but it lacks information
|
||||
for load quality. If you have CKAN deployed in your organisation and wish
|
||||
for load quality. If you have CKAN deployed in your organization and wish
|
||||
exhange or process the data referenced in CKAN, then there are synergies between
|
||||
CKAN and marti.
|
||||
|
||||
Samples exist on CKAN integration.
|
||||
|
||||
Magda and martiLQ
|
||||
-----------------
|
||||
## Magda and martiLQ
|
||||
|
||||
Another source of data is [Magda](https://magda.io/) which has API metadata
|
||||
definitions. Magda is more about data federation and as such provides
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
Custom Definition
|
||||
=================
|
||||
|
||||
The custome definition section allows the inclusion of extensions
|
||||
The custom definition section allows the inclusion of extensions
|
||||
to the standard. To demonstrate the inclusion, there are three
|
||||
sample extensions. These are:
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 157 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 122 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 127 KiB |
|
|
@ -1,5 +1,5 @@
|
|||
Magda definitions
|
||||
=================
|
||||
# Magda definitions
|
||||
|
||||
|
||||
https://magda.io/
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ modified|Modified date and time of the **martiLQ** document|Now
|
|||
tags|List of tags or keywords|
|
||||
publisher|Publisher name|
|
||||
contactPoint|Contact point of a person or team|
|
||||
accessLevel|Acces level|
|
||||
accessLevel|Access level|
|
||||
rights|Rights|
|
||||
|
||||
* Batch
|
||||
|
|
@ -54,7 +54,7 @@ rights|Rights|
|
|||
### Information extension
|
||||
|
||||
The information supplied can be extended by party agreement and there
|
||||
are place holders in the defintion.
|
||||
are place holders in the definition.
|
||||
|
||||
## Resource
|
||||
|
||||
|
|
@ -62,7 +62,7 @@ The resource section is a list of documents or files that are to be grouped
|
|||
together are listed under the same **martiLQ** definition.
|
||||
|
||||
At least one document or file must be included. If the same resource is repeated
|
||||
it will commonly be for definiting multiple formats, with each file having a
|
||||
it will commonly be defining multiple formats, with each file having a
|
||||
different extension. Commonly the definition includes at least the following
|
||||
items:
|
||||
|
||||
|
|
@ -84,7 +84,7 @@ for more details
|
|||
Name|Description|Default or values
|
||||
---|---|--
|
||||
hash|Hash of document - The hash of the document, which can be blank especially for large documents
|
||||
algo|Hash algorithm - Algoroithm used to generate the hash value or sign it
|
||||
algo|Hash algorithm - Algorithm used to generate the hash value or sign it
|
||||
description|Description - A more detailed description
|
||||
version|Version - A document version
|
||||
encoding|Encoding
|
||||
|
|
@ -117,6 +117,7 @@ sample can be generated using the GOLANG client program with parameters:
|
|||
"tags": null,
|
||||
"license": "",
|
||||
"state": "active",
|
||||
"stateModified": "2021-11-02T22:44:29.6887001+11:00",
|
||||
"batch": 1.001,
|
||||
"describedBy": "",
|
||||
"landingPage": "",
|
||||
|
|
@ -130,6 +131,7 @@ sample can be generated using the GOLANG client program with parameters:
|
|||
"modified": "2021-11-02T07:47:13.9410018+11:00",
|
||||
"expires": "2023-11-02T00:00:00+11:00",
|
||||
"state": "active",
|
||||
"stateModified": "2021-11-02T22:44:29.6881663+11:00",
|
||||
"author": "",
|
||||
"length": 3654,
|
||||
"hash": {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ load quality metrics.
|
|||
* Number of records in the document - This is the number of data primary records not the
|
||||
count of end of lines and is agreed between parties. XML record counts could be based
|
||||
on the number of primary segments under root. JSON records can be counted in a similar way.
|
||||
The headers or trailling records are not counted
|
||||
The headers or trailing records are not counted
|
||||
|
||||
## Addresses deficiencies
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# References
|
||||
|
||||
The following are references to documents that inspired the creation of **martiLQ**
|
||||
document and associatd framework.
|
||||
document and associated framework.
|
||||
|
||||
https://dex.dss.gov.au/sites/default/files/documents/2021-06/data-exchange-protocols-june-2021.pdf
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ for various programming languages and situations. As many programming languages
|
|||
generate portable programs that can execute on multiple operating systems, the
|
||||
likelihood is that a tools exists for you.
|
||||
|
||||
The source for tools is provided in the Github repository and some have precompiled
|
||||
The source for tools is provided in the Github repository and some have pre-compiled
|
||||
images.
|
||||
|
||||
See the project source directory for more details.
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
Who is likely to use martiLQ
|
||||
============================
|
||||
# Who is likely to use martiLQ
|
||||
|
||||
You are likely to find the **martiLQ** framework relevant if you:
|
||||
|
||||
1. Have many document exchanges, such as End of Day batches
|
||||
2. Need to verify or reconcile the documents
|
||||
|
||||
Data exchanges
|
||||
--------------
|
||||
## Data exchanges
|
||||
|
||||
If you are creating or receiving many documents or files on a regular basis
|
||||
then you probably have some framework defined. The framework may be as simple as:
|
||||
|
|
@ -24,8 +22,7 @@ Simple framework such as the above have limitations, such as:
|
|||
* Lower automation prospects and alignment to DataSecOps
|
||||
* Poor fit to web applications (they tend to be designed for FTP and LAN)
|
||||
|
||||
Framework Sidecar files
|
||||
-----------------------
|
||||
## Framework Sidecar files
|
||||
|
||||
The **martiLQ** framework addresses the issues and limitations by using sidecar
|
||||
or shadow files. The [concept of sidecar files](https://en.wikipedia.org/wiki/Sidecar_file) is
|
||||
|
|
@ -35,7 +32,7 @@ Sidecar files can also be implemented as ``forks`` and built into the operating
|
|||
in Mac OS X HFS. The Microsoft NTFS supports Alternate Data Streams to achieve a similar outcome.
|
||||
Unfortunately this information is not transferrable to other systems.
|
||||
|
||||
The proposition is to define a format for the sidecare file and provide common library tools that
|
||||
The proposition is to define a format for the sidecar file and provide common library tools that
|
||||
can be be used on multiple platforms when exchanging documents / files. Multiple documents can be
|
||||
defined in a singel **martiLQ** definition which adds to efficiency and productivity if used
|
||||
defined in a single **martiLQ** definition which adds to efficiency and productivity if used
|
||||
for End of Day or similar batches - or even single file transfers.
|
||||
|
|
|
|||
|
|
@ -37,3 +37,10 @@ signKey_Password =
|
|||
proxy =
|
||||
username =
|
||||
password =
|
||||
|
||||
[Smtp]
|
||||
|
||||
host =
|
||||
port =
|
||||
username =
|
||||
password =
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
. .\source\powershell\MartiLQAttribute.ps1
|
||||
|
||||
|
||||
function New-MartiDefinition
|
||||
function New-MartiLQDefinition
|
||||
{
|
||||
Param(
|
||||
[String] $ConfigPath = $null
|
||||
|
|
@ -23,13 +23,6 @@ function New-MartiDefinition
|
|||
renderer = "MARTILQREFERENCE:Mustache"
|
||||
url = ""
|
||||
}
|
||||
|
||||
$oAcknowledgement = [PSCustomObject]@{
|
||||
url = ""
|
||||
algo = ""
|
||||
value = ""
|
||||
signed = $false
|
||||
}
|
||||
|
||||
|
||||
if ($null -eq $ConfigPath -or $ConfigPath -eq "") {
|
||||
|
|
@ -97,7 +90,7 @@ function New-MartiDefinition
|
|||
publisher = $publisher
|
||||
contactPoint = $oConfig.contactPoint
|
||||
accessLevel = $oConfig.accessLevel
|
||||
consumer = $lconsumer
|
||||
consumers = $lconsumer
|
||||
rights = $oConfig.rights
|
||||
license = $oConfig.license
|
||||
state = $oConfig.state
|
||||
|
|
@ -108,7 +101,7 @@ function New-MartiDefinition
|
|||
theme =$oConfig.theme
|
||||
|
||||
resources = $lresource
|
||||
acknowledge = $oAcknowledgement
|
||||
acknowledge = Get-Acknowledgement
|
||||
custom = $lCustom
|
||||
}
|
||||
|
||||
|
|
@ -116,6 +109,94 @@ function New-MartiDefinition
|
|||
}
|
||||
|
||||
|
||||
function Save-MartiLQDefinition
|
||||
{
|
||||
Param(
|
||||
[Parameter(Mandatory)][PSCustomObject] $MartiLQ,
|
||||
[Parameter(Mandatory)][String] $FilePath
|
||||
)
|
||||
|
||||
$fileJson = $FilePath
|
||||
$MartiLQ | ConvertTo-Json -depth 100 | Out-File $fileJson
|
||||
|
||||
return $fileJson
|
||||
}
|
||||
|
||||
|
||||
function Restore-MartiLQDefinition
|
||||
{
|
||||
Param(
|
||||
[Parameter(Mandatory)][String] $FilePath
|
||||
)
|
||||
|
||||
$oMartiLQ = [PSCustomObject](Get-Content -Raw $FilePath | Out-String | ConvertFrom-Json)
|
||||
|
||||
$version = Get-DefinitionVersion -MartiLQ $oMartiLQ
|
||||
if ($version -lt "0.0.2") {
|
||||
if () {
|
||||
[System.Collections.ArrayList]$lconsumer = @()
|
||||
$oMartiLQ | Add-Member -Name "consumers" -Type NoteProperty -Value $lconsumer
|
||||
}
|
||||
|
||||
if () {
|
||||
$oMartiLQ | Add-Member -Name "acknowledge" -Type NoteProperty -Value (Get-Acknowledgement)
|
||||
}
|
||||
|
||||
Set-DefinitionVersion -MartiLQ $oMartiLQ -Version "0.0.2"
|
||||
$newVersion = Get-DefinitionVersion -MartiLQ $oMartiLQ
|
||||
Write-Host "Updating from version $version to $newVersion"
|
||||
}
|
||||
|
||||
return $oMartiLQ
|
||||
}
|
||||
|
||||
function Get-DefinitionVersion
|
||||
{
|
||||
param (
|
||||
[Parameter(Mandatory)][PSCustomObject] $MartiLQ
|
||||
)
|
||||
|
||||
$version = "0.0.1"
|
||||
$MartiLQ.custom | ForEach-Object {
|
||||
if ($_.extension -eq "software" -and $_.softwareName -eq (Get-SoftwareName)) {
|
||||
$version = $_.version
|
||||
}
|
||||
}
|
||||
|
||||
return $version
|
||||
}
|
||||
|
||||
function Set-DefinitionVersion
|
||||
{
|
||||
param (
|
||||
[Parameter(Mandatory)][PSCustomObject] $MartiLQ,
|
||||
[Parameter(Mandatory)][String] $Version
|
||||
)
|
||||
|
||||
$MartiLQ.custom | ForEach-Object {
|
||||
|
||||
if ($_.extension -eq "software" -and $_.softwareName -eq (Get-SoftwareName)) {
|
||||
$_.version = $Version
|
||||
#return $MartiLQ
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#return $MartiLQ
|
||||
}
|
||||
|
||||
function Get-Acknowledgement{
|
||||
|
||||
$oAcknowledgement = [PSCustomObject]@{
|
||||
url = ""
|
||||
algo = ""
|
||||
value = ""
|
||||
signed = $false
|
||||
}
|
||||
|
||||
return $oAcknowledgement
|
||||
}
|
||||
|
||||
function Set-MartiAttribute
|
||||
{
|
||||
Param(
|
||||
|
|
@ -244,9 +325,6 @@ function Get-MartiResource {
|
|||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
function ConvertFrom-Ckan
|
||||
{
|
||||
Param(
|
||||
|
|
@ -341,7 +419,6 @@ Param(
|
|||
}
|
||||
|
||||
|
||||
|
||||
function Compress-MartiLQ
|
||||
{
|
||||
Param(
|
||||
|
|
@ -397,6 +474,3 @@ Param(
|
|||
Close-Log
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
|
||||
|
||||
|
||||
$script:SoftwareVersion = "0.0.1"
|
||||
$script:SoftwareVersion = "0.0.2"
|
||||
$global:default_metaFile = "##martilq##.json"
|
||||
|
||||
$global:martiLQ_configuration = $null
|
||||
|
|
@ -58,6 +58,11 @@ function Get-DefaultConfiguration {
|
|||
proxy_User = $null
|
||||
proxy_Credential = $null
|
||||
|
||||
smtp_Host = $null
|
||||
smtp_Port = $null
|
||||
smtp_User = $null
|
||||
smtp_Credential = $null
|
||||
|
||||
loaded = $false
|
||||
}
|
||||
|
||||
|
|
@ -155,6 +160,13 @@ function Import-Configuration {
|
|||
$oConfig.hashAlgorithm = Set-ConfigurationValue $oConfig.hashAlgorithm -Value $iConfig.Hash.hashAlgorithm
|
||||
}
|
||||
|
||||
if ($null -ne $iConfig.Smtp) {
|
||||
$oConfig.smtp_Host = Set-ConfigurationValue $oConfig.smtp_Host -Value $iConfig.Smtp.host
|
||||
$oConfig.smtp_Port = Set-ConfigurationValue $oConfig.smtp_Port -Value $iConfig.Smtp.port
|
||||
$oConfig.smtp_User = Set-ConfigurationValue $oConfig.smtp_Port -Value $iConfig.Smtp.username
|
||||
$oConfig.smtp_Credential = Set-ConfigurationValue $oConfig.smtp_Credential -Value $iConfig.Smtp.password
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
$global:martiLQ_configuration = $oConfig
|
||||
|
|
|
|||
|
|
@ -0,0 +1,186 @@
|
|||
|
||||
. .\source\powershell\MartiLQ.ps1
|
||||
. .\source\powershell\MartiLQConfiguration.ps1
|
||||
|
||||
|
||||
function Send-EmailAck {
|
||||
param (
|
||||
[String] $FileAttachment,
|
||||
[String] $Recipient,
|
||||
[String] $State,
|
||||
[int] $Buffersize = 1024
|
||||
)
|
||||
|
||||
$receiver = $Recipient.Substring(5)
|
||||
Write-Host "Sending acknowledgment via email to: $receiver " -ForegroundColor Green
|
||||
|
||||
$EmailFrom = $env:MARTILQ_EMAIL_FROM
|
||||
|
||||
$Subject = "martiLQ acknowledge [$State]"
|
||||
$Body = "Simple email ack"
|
||||
|
||||
$password = ConvertTo-SecureString $env:MARTILQ_EMAIL_PASSWORD -AsPlainText -Force
|
||||
$credential = New-Object System.Management.Automation.PSCredential ($env:MARTILQ_EMAIL_USERNAME, $password)
|
||||
|
||||
Write-Host "SMTP: $($env:MARTILQ_EMAIL_HOST) :: 465 " -ForegroundColor Yellow
|
||||
Write-Host "Send with: $FileAttachment :: $Subject :: $Body " -ForegroundColor Yellow
|
||||
|
||||
$att = new-object Net.Mail.Attachment($FileAttachment)
|
||||
|
||||
#Send mail with attachment
|
||||
$from = New-Object System.Net.Mail.MailAddress($EmailFrom)
|
||||
$to = New-Object System.Net.Mail.MailAddress($receiver)
|
||||
$email = New-Object System.Net.Mail.Mailmessage($from, $to)
|
||||
$email.Subject = $Subject
|
||||
$email.Body = $Body
|
||||
$email.IsBodyHTML = $true
|
||||
$email.Attachments.Add($att)
|
||||
|
||||
$att.Dispose()
|
||||
|
||||
$smtp = New-Object Net.Mail.SmtpClient($env:MARTILQ_EMAIL_HOST, 465)
|
||||
$smtp.EnableSSL = $true
|
||||
$smtp.Credentials = New-Object System.Net.NetworkCredential($env:MARTILQ_EMAIL_USERNAME, $env:MARTILQ_EMAIL_PASSWORD)
|
||||
$smtp.Send($email)
|
||||
|
||||
}
|
||||
|
||||
function Get-Resources {
|
||||
param (
|
||||
[Parameter(Mandatory)][PSCustomObject] $MartiLQ,
|
||||
[String] $DataPath,
|
||||
[String] $CurrentState,
|
||||
[String] $Consumer
|
||||
)
|
||||
|
||||
$nextState = "received"
|
||||
|
||||
Try {
|
||||
foreach ($item in $MartiLQ.resources) {
|
||||
if ($item.state -eq $currentState) {
|
||||
|
||||
$item.state = $nextState
|
||||
}
|
||||
}
|
||||
|
||||
if ($Consumer -ne "") {
|
||||
[System.Collections.ArrayList]$lconsumer = @()
|
||||
$lconsumer += $Consumer
|
||||
$MartiLQ.consumers = $lconsumer
|
||||
}
|
||||
|
||||
$today = Get-Date
|
||||
$dateToday = $today.Tostring("yyyy-MM-ddTHH:mm:ss")
|
||||
$MartiLQ.stateModified = $dateToday
|
||||
$MartiLQ.state = $nextState
|
||||
|
||||
# Notification
|
||||
$ack = $MartiLQ.acknowledge
|
||||
|
||||
if ($ack.url.startswith("mail:"))
|
||||
{
|
||||
$fileJson = "./temp/MartiBSBRemote_interim2.json"
|
||||
$attachment = Save-MartiLQDefinition -MartiLQ $oMarti -FilePath $fileJson
|
||||
Send-EmailAck -FileAttachment $attachment -Recipient $ack.url -State $nextState
|
||||
}
|
||||
|
||||
} Catch {
|
||||
Write-Host "Error in resource get: $_"
|
||||
}
|
||||
|
||||
return $MartiLQ
|
||||
}
|
||||
|
||||
|
||||
function Test-Resource {
|
||||
param (
|
||||
[Parameter(Mandatory)][PSCustomObject] $MartiLQ,
|
||||
[String] $DataPath,
|
||||
[String] $CurrentState,
|
||||
[String] $Consumer
|
||||
)
|
||||
|
||||
$nextState = "verified"
|
||||
|
||||
Try {
|
||||
foreach ($item in $MartiLQ.resources) {
|
||||
if ($item.state -eq $currentState) {
|
||||
|
||||
$item.state = $nextState
|
||||
}
|
||||
}
|
||||
|
||||
if ($Consumer -ne "") {
|
||||
[System.Collections.ArrayList]$lconsumer = @()
|
||||
$lconsumer += $Consumer
|
||||
$MartiLQ.consumers = $lconsumer
|
||||
}
|
||||
|
||||
$MartiLQ.state = $nextState
|
||||
} Catch {
|
||||
Write-Host "Error in resource test: $_"
|
||||
}
|
||||
|
||||
return $MartiLQ
|
||||
}
|
||||
|
||||
function Invoke-ProcessResource {
|
||||
param (
|
||||
[Parameter(Mandatory)][PSCustomObject] $MartiLQ,
|
||||
[String] $DataPath,
|
||||
[String] $CurrentState,
|
||||
[String] $Consumer
|
||||
)
|
||||
|
||||
$nextState = "processed"
|
||||
|
||||
Try {
|
||||
foreach ($item in $MartiLQ.resources) {
|
||||
if ($item.state -eq $currentState) {
|
||||
|
||||
$item.state = $nextState
|
||||
}
|
||||
}
|
||||
|
||||
if ($Consumer -ne "") {
|
||||
[System.Collections.ArrayList]$lconsumer = @()
|
||||
$lconsumer += $Consumer
|
||||
$MartiLQ.consumers = $lconsumer
|
||||
}
|
||||
|
||||
$MartiLQ.state = $nextState
|
||||
} Catch {
|
||||
Write-Host "Error in resource process: $_"
|
||||
}
|
||||
|
||||
return $MartiLQ
|
||||
}
|
||||
|
||||
$currentState = "expired"
|
||||
$nextState = "active"
|
||||
$consumer = "Test-Framework"
|
||||
|
||||
$fileJson = "C:\Users\meerkat\source\marti\docs\source\samples\powershell\test\MartiBSBRemote.json"
|
||||
$oMarti = Restore-MartiLQDefinition -FilePath $fileJson
|
||||
|
||||
$oMarti.acknowledge.url = "mail:tp_reklam@villacentrum.com"
|
||||
|
||||
$today = Get-Date
|
||||
$dateToday = $today.Tostring("yyyy-MM-ddTHH:mm:ss")
|
||||
$oMarti.stateModified = $dateToday
|
||||
$oMarti.state = $nextState
|
||||
|
||||
foreach ($item in $oMarti.resources) {
|
||||
if ($item.state -eq $currentState) {
|
||||
$item.state = $nextState
|
||||
}
|
||||
}
|
||||
|
||||
$oMarti = Get-Resources -MartiLQ $oMarti[0] -DataPath "" -CurrentState "active" -Consumer $consumer
|
||||
|
||||
$oMarti = Test-Resource -MartiLQ $oMarti -DataPath "" -CurrentState "received" -Consumer $consumer
|
||||
|
||||
$oMarti = Invoke-ProcessResource -MartiLQ $oMarti -DataPath "" -CurrentState "verified" -Consumer $consumer
|
||||
|
||||
$fileJson = "C:\Users\meerkat\source\marti\docs\source\samples\powershell\test\MartiBSBRemote_v2.json"
|
||||
Save-MartiLQDefinition -MartiLQ $oMarti -FilePath $fileJson
|
||||
|
|
@ -91,4 +91,4 @@ function New-LocalTempFile{
|
|||
}
|
||||
|
||||
return Join-Path -Path $temp_dir -ChildPath $doc_name
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,14 +24,13 @@ from mutility import mUtility
|
|||
|
||||
class martiLQ:
|
||||
|
||||
_SoftwareVersion = "0.0.1"
|
||||
_default_metaFile = "##martilq##.json"
|
||||
|
||||
_oSoftware = {
|
||||
"extension": "software",
|
||||
"softwareName": "MARTILQREFERENCE",
|
||||
"softwareName": mConfiguration.GetSoftwareName(),
|
||||
"author": "Meerkat@merebox.com",
|
||||
"version": "0.0.1"
|
||||
"version": mConfiguration.GetSoftwareVersion()
|
||||
}
|
||||
|
||||
_oTemplate = {
|
||||
|
|
|
|||
|
|
@ -19,24 +19,26 @@ from mlogging import mLogging
|
|||
|
||||
class mConfiguration:
|
||||
|
||||
_SoftwareVersion = "0.0.1"
|
||||
_SoftwareVersion = "0.0.2"
|
||||
_default_metaFile = "##martilq##.json"
|
||||
|
||||
_oSoftware = {
|
||||
"extension": "software",
|
||||
"softwareName": "MARTILQREFERENCE",
|
||||
"softwareName": GetSoftwareName(),
|
||||
"author": "Meerkat@merebox.com",
|
||||
"version": "0.0.1"
|
||||
"version": GetSoftwareVersion()
|
||||
}
|
||||
|
||||
|
||||
_oConfiguration = None
|
||||
_Log = None
|
||||
|
||||
|
||||
def GetSoftwareName(self):
|
||||
def GetSoftwareName():
|
||||
return "MARTILQREFERENCE"
|
||||
|
||||
def GetSoftwareVersion():
|
||||
return _SoftwareVersion
|
||||
|
||||
def __init__(self):
|
||||
|
||||
self._oConfiguration = {
|
||||
|
|
|
|||
Loading…
Reference in New Issue