Refactoring for documentation

Included Sphinx

.gitignore

@@ -11,9 +11,13 @@ buildNumber.properties
 .mvn/wrapper/maven-wrapper.jar
 
 test/powershell/results/
-docs/samples/powershell/test/
 
-docs/samples/python/test
+docs/source/samples/powershell/test/
+docs/source/samples/python/test
+
+docs/build
+
 source/python/client/__pycache__
 
-source/marti/source/golang/client/src/test
+source/golang/client/src/test
+

README.md

@@ -40,7 +40,7 @@ information
 ## Transfer information
 
 The information in the **martiLQ** document is summarised below. For more detailed
-information see [martiLQ definition](martiLQ.md)
+information see [martiLQ definition](/docs/source/martiLQ.md)
 
 ### Mandatory information
 
@@ -48,7 +48,7 @@ The mandatory information is:
 
 * Title
 * Unique identifier
-* Distribution list - See Distribution section summary below or detailed document [Distribution](docs/distribution.md)
+* Resource list - See Resource section summary below or detailed document [Resource](docs/source/resource.md)
 
 ### Optional information
 
@@ -75,11 +75,11 @@ The optional information is:
 The information supplied can be extended by party agreement and there
 are place holders in the defintion.
 
-### Distribution 
+### Resource 
 
-The distribution section is intended to allow multiple data files to be
+The resource section is intended to allow multiple data files to be
-grouped together.  The distribution section can be repeated, but at least 
+grouped together.  The resource section can be repeated, but at least 
-one must be included.  If the distribution is repeated it will commonly 
+one must be included.  If the resource is repeated it will commonly 
 be for definiting multiple formats of the same data or batching of
 different data together from the same extract process.
 
@@ -92,9 +92,9 @@ different data together from the same extract process.
 * Hash of document - The hash of the document, which can be blank especially for large documents
 * Hash algorithm
 
-### Distribution optional
+### Resource optional
 
-The following are some of the optional items in the distribution section.  See [Distribution](docs/distribution.md)
+The following are some of the optional items in the resource section.  See [Resource](docs/source/resources.md)
 for more details
 
 * Description

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/ckan.md

@@ -1,5 +0,0 @@
-
-
-https://ckan.org/
-
-Sample Json from https://data.gov.au/data/dataset/f2b7c2c1-f4ef-4ae9-aba5-45c19e4d3038 

docs/ckan.txt

@@ -1 +0,0 @@
-

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/quality.md

@@ -1,60 +0,0 @@
-# Quality definition
-
-The **martiLQ** definition allows for the inclusion of a load quality 
-definition.  This load quality definition is intended to be
-able to be applied universally with common tools.  As such not 
-all needs are covered.
-
-## Defined laod quality metrics
-
-
-* Number of records in the document - This is the number of data primary records not the 
-   count of end of lines and is agreed between parties.  XML record counts could be based 
-   on the number of primary segments under root.  JSON records can be counted in a similar way.
-   The headers or trailling records are not counted
-
-
-Sequence number - linked to the job producing the document and therefore a daily, weekly and monthly extracts for the 
-same document would have different sequence numbers
-
-Discourage us of magic formats for document names such as
-
-XT_PARTY_20210911_SQ00001_N000567891234_V01.DAT
-
-Header and Trailer records are not part of the quality definition except
-
-In fact the trailer record is intended to be replaced by this quality definition
-
-The header, if it exists, is only used where it identifies the column name sequence of the data.
-
-Effective and Process date -----------------
-
-Row count - 
-Column count -
-Depth count - 
-
-Mandatory - data must be present and cannot be blank or null
-Uniqueness - data value must be unique within the document
-Data integrity - data exists within defined tolerances 
-
-
-row count 9999
-
-column count 9999
-
-column_name sum 9999
-
-column_name gt 9999
-column_name lt 9999
-column_name eq 9999
-column_name eq "  "
-column_name ne 9999
-column_name ne "  "    == Check for value
-column_name ge 9999
-column_name le "  "
-column_name in "  ", " ", " "
-
-column_name is integer
-column_name is decimal
-column_name is unique
-

docs/source/README.md

@@ -21,4 +21,4 @@ can adjust if they resonate with your circumstances,
 6. [Comparison of martiLQ definition](comparison.md)
 7. [References](references.md)
 
-[!INCLUDE [martiLQ High Level Definition](../martiLQ.md)]
+[!INCLUDE [martiLQ High Level Definition](martiLQ.md)]

docs/source/attributes.md

@@ -0,0 +1,32 @@
+# Attribute definition
+
+A Resource can list attributes related to the document / file.
+
+An attribute is a generic definition and conventions are 
+observed in the definitions that are captured here.
+
+## Attribute definition 
+
+The Attribute consists of:
+
+   * category - A value of "dataset",
+   * name
+   * function - A value such as "count"
+   * comparison - A comparisn value or NA.  Values are "EQ", 
+        "NE", "GT", "LT
+   * value - The value for the attribute based on the above complex key, excluding comparison
+
+A sample JSON is shown below which describes the 
+number of records in the file for the given format.
+
+```json
+    "attributes": [
+        {
+            "category": "dataset",
+            "name": "records",
+            "function": "count",
+            "comparison": "EQ",
+            "value": "9"
+        }
+    ]
+```

docs/source/ckan.md

@@ -0,0 +1,15 @@
+# CKAN definition
+
+The **martiLQ** has used similar terms and structures that are found in the 
+CKAN API describing the resources.  This similarity allows for simple mapping of values
+from the CKAN format to **martiLQ** format.
+
+What **martiLQ** brings above the CKAN definition is attributes that can be
+used to reconcile with the recieved data, plus the ability to define 
+compressed and encrypted resources.
+
+For more information on CKAN see https://ckan.org/
+
+A sample Json to compare against the **martiLQ** definition 
+is https://data.gov.au/data/dataset/f2b7c2c1-f4ef-4ae9-aba5-45c19e4d3038
+

docs/source/comparison.md

@@ -34,10 +34,10 @@ Samples exist on CKAN integration.
 ## Magda and martiLQ
 
 Another source of data is [Magda](https://magda.io/) which has API metadata
-definitions.  Magda is more about data fedaration and as such provides
+definitions.  Magda is more about data federation and as such provides
 functionality on finding data sources and describing the contents.
 
 The Magda software is able to generate APIs and data content.  This does not 
 address the needs of data processing pipeline when reconciliation is required.
 
-If you have Magda data sources then synergies exist between Magda and marti.
+If you have Magda data sources then synergies exist between Magda and martiLQ.

docs/source/conf.py

@@ -0,0 +1,64 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'MartiLQ'
+copyright = '2021, meerkat@merebox.coom'
+author = 'meerkat@merebox.coom'
+
+# The full version, including alpha/beta/rc tags
+release = '0.0.1'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['myst_parser']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = [
+    'samples/python/test/*',  
+    'samples/powershell/test/*',  
+    ]
+
+source_suffix = {
+    '.rst': 'restructuredtext',
+    '.txt': 'restructuredtext',
+    '.md': 'markdown',
+}
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+#html_theme = 'alabaster'
+html_theme = 'sphinx_rtd_theme'
+
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']

docs/source/custom.md

@@ -0,0 +1,9 @@
+# Custom Definition
+
+The custome definition section allows the inclusion of extensions
+to the standard.  To demonstrate the inclusion, there are three
+sample extensions.  These are:
+
+  * Software - describing the **martiLQ** software version
+  * Spatial - Defining the geographical boundary of the documents
+  * Temporal - Defining date and time aspects of the documents

docs/source/index.rst

@@ -0,0 +1,37 @@
+.. MartiLQ documentation master file, created by
+   sphinx-quickstart on Sun Oct 31 14:30:06 2021.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to MartiLQ's documentation!
+===================================
+
+.. toctree::
+   :hidden:
+   :maxdepth: 2
+   
+   objective.md
+   martiLQ.md
+   why.md
+   what.md
+   when.md
+   who.md
+   resources.md
+   attributes.md
+   custom.md
+   quality.md
+   comparison.md
+   ckan.md
+   magda.md
+   references.md
+
+**martiLQ** stands for metadata reconcilation for transfer information, load quality.
+
+Before starting with **martiLQ** it is advisable to understand if it is right for
+your organisation's needs.  Information is available in a number of short 
+documents.  Start with the (objectives)[objective.md]
+
+There are sample implementations which you
+can adjust if they resonate with your circumstances.
+
+The source, documentation and samples are available at `<https://github.com/meerkat-manor/marti>`_

docs/source/magda.md

@@ -1,4 +1,4 @@
-
+# Magda definitions
 
 https://magda.io/ 
 

docs/source/martiLQ.md

@@ -0,0 +1,75 @@
+# MartiLQ document
+
+The metadata reconciliation transfer information is referred
+to as the **martiLQ** document throughout this documentation.
+
+The **martiLQ** document can be part of a message or a file
+in its own right. The definition is curently a Json file.
+
+
+
+### Mandatory information
+
+The mandatory information is:
+
+* Title
+* Unique identifier
+* Resource list - See Resource section summary below or detailed document [Resource](resources.md)
+
+### Optional information
+
+The optional information is:
+
+* Description
+* Modified
+* Tags or keywords
+* Publisher
+* Contact point
+* Acces level
+* Rights
+* Batch
+* License
+* Described By - A link to the metadata describing the document.  
+   More detailed information could be supplied at the link
+* Landing page
+* Theme
+* Custom list - List of custom entries, one being the **martiLQ** software details
+  see [custom](custom.md)
+
+### Information extension
+
+The information supplied can be extended by party agreement and there
+are place holders in the defintion.
+
+### Resource 
+
+The resource section is a list of documents or files that are to be grouped
+together are listed under the same **martiLQ** definition.
+
+At least one document or file must be included.  If the same resource is repeated 
+it will commonly be for definiting multiple formats, with each file having a 
+different extension.  Commonly the definition includes at least the following
+items:
+
+* Title - A summary description of the document contents
+* Unique identifier - A unique identifier, commonly a GUID
+* Document name - A name of the document such as thefile name
+* Issued date - When the document was made available. The date can include time 
+* Modified - When the document was created or modified.  This is the data and time
+* Size of document - The document size in bytes
+* URL - This can be ``file://``, ``https://``, ``ftp://``, etc resource location
+
+### Resource optional
+
+The following are some of the optional items in the resource section.  See [Resource](resources.md)
+for more details
+
+* Hash of document - The hash of the document, which can be blank especially for large documents
+* Hash algorithm - Algoroithm used to generate the hash value or sign it
+* Description - A more detailed description 
+* Version - A document version
+* Encoding
+* Content Type 
+* Compression
+* Encryption
+* Author

docs/source/objective.md

@@ -0,0 +1,22 @@
+# MartiLQ objective
+
+The objective of **martLQ** is to define a simle standard for
+capturing the data files being transferred.  It is not for 
+real time web service transactions.  
+
+**martiLQ** is about file and document transfer and reconciling
+that the all files have arrived and have not changed, and if so
+required are also encrypted.
+
+The proposition is to have a common, machine readable format
+for file exchange that:
+
+  * ensures data load quality and reconciles
+  * can be used on Linux or Windows or Mac
+  * can be used with Python, Java, PowerShell, Golang, etc 
+  * can be used by web services
+  * uses a text based format (JSON)
+  * can form part of the data processing pipeline
+
+And finally is easy to understand.
+

docs/source/quality.md

@@ -0,0 +1,47 @@
+# Quality definition
+
+The **martiLQ** definition allows for the inclusion of load quality 
+definitions.  The load quality definition is intended to be
+able to be applied universally with common tools.  Not 
+all needs are covered with the base definition but can be extended.
+
+## Defined load quality metrics
+
+* Sequential batch number - This is a decimal number defined at the **martiLQ** definition
+   and applies to all resources.  The integer portion is for new batches and the fraction
+   part can be used for issues with the same data extract.  such as requiring resend because
+   a resource was missing.
+
+* Number of records in the document - This is the number of data primary records not the 
+   count of end of lines and is agreed between parties.  XML record counts could be based 
+   on the number of primary segments under root.  JSON records can be counted in a similar way.
+   The headers or trailling records are not counted
+
+## Addresses deficiencies 
+
+The **martiLQ** objective is to address deficiencies with alternative 
+data load quality approaches such as:
+
+ * magic formats in file names
+ * identifying the number of files
+ * knowing when all files are ready
+ * separate documentation that is unlinked
+ * securing the data
+ * adding footers to the data, requiring custom file handlers
+
+## Extending load quality metrics 
+
+**martLQ** framewokr and importantly is open to extension so that extra
+load metrics appropriate to the situation can be included.
+
+### Extension ideas
+
+The following extensions for load quality can easily be included:
+
+  * Mandatory data in column
+  * Uniqueness of data values 
+  * Data values are within defined tolerances
+  * Check for data exclusions
+
+And all this information is included in the **martiLQ** definition
+allowing for self describing load quality.

docs/source/references.md

@@ -1,4 +1,4 @@
-
+# References
 
 
 https://dex.dss.gov.au/sites/default/files/documents/2021-06/data-exchange-protocols-june-2021.pdf

docs/source/resources.md

@@ -1,6 +1,6 @@
-# Distribution definition
+# Resources definition
 
-The distrubution section defines the files that are grouped
+The resources section defines the files that are grouped
 together by association.  This association is not defined but can
 include different formats of the same data or a common batch extract 
 such as end of day.
@@ -10,7 +10,7 @@ compressed with a utility such as WinZIP or 7ZIP.  In the situation
 where a ZIP file expands to multiple documents, then the expectation is
 that the ZIP file contains a **martiLQ** document describing its contents.
 
-The elements in the distribution section are:
+The elements in the resource section are:
 
 * Title
 * Document name - Commonly being absolute or relative file name.
@@ -20,8 +20,9 @@ The elements in the distribution section are:
 * Size of file - The file size in bytes
 * Hash of file - The hash of the file, which can be blank especially for large files
 * Hash algorithm
+* Attributes - List of attributes associated with the document
 
-The following are optional in the distribution section.
+The following are optional in the resource section.
 
 * Identifier
 * Description
@@ -29,8 +30,7 @@ The following are optional in the distribution section.
 * Version - File version.  The same file could be updated or this might denote the next version
     of a regular report.  For example a daily extract will have the version number incremented
     every day and provide a new URL.  The previous file can be retained.
-* Format - if not specified then the consumer will in all likelihood use the file extension / mime type
+* Content type - if not specified then the consumer will in all likelihood use the file extension / mime type
-* Media Type
 * Expiry Date - The date and time that this file expires and can be removed from the download URL
     location.  This is not the file retention period as might be required for archiving.
 * Described By - A link to the metadata describing this file data and format

docs/source/what.md

@@ -1,6 +1,6 @@
 # What is marti
 
-The foundation pillar for the **martiLQ** framework is the [martiLQ document](../martiLQ.md) 
+The foundation pillar for the **martiLQ** framework is the [martiLQ document](martiLQ.md) 
 that defines the reconciliation and other metadata of the document / file being transferred.
 
 A definition, while fundamental, benefits from having tools that can create, read and 
@@ -12,7 +12,7 @@ for various programming languages and situations.  As many programming languages
 generate portable programs that can execute on multiple operating systems, the
 likelihood is that a tools exists for you.
 
-The source for tools is provided in this repository and some have precompiled
+The source for tools is provided in the Github repository and some have precompiled
 images.
 
-See the [source](../source/) directory for more details.
+See the project source directory for more details.

docs/source/when.md

@@ -1,6 +1,7 @@
 # When would you use martiLQ
 
 You are likely to start using the **martiLQ** framework when:
+
 1. you have no existing standard or framework or;
 2. your existing standards no longer meets you needs or;
 3. you are starting document exchange with another business or division that uses the framework or;
@@ -8,7 +9,7 @@ You are likely to start using the **martiLQ** framework when:
 
 If you already have a standard and it works for you, and you have no upcoming (large)
 initiative that would benefit from the framework, then stick with what you have.  The benefits
-of the framework are unlikely to weigh in your framework
+of the framework are unlikely to weigh in your framework.
 
 ## Read the material
 
@@ -19,6 +20,3 @@ your direction will more than likely add to your support costs.
 **Note**: As the framework is based on open structure and can be processed independently 
 without the tools, you can export the definitions to other tools / future methods.
 
-See also:
-1. [What the framework contains](what.md)
-2. [Quality definition](quality.md)

docs/source/who.md

@@ -1,8 +1,8 @@
-# Who is likely to use marti
+# Who is likely to use martiLQ
 
 You are likely to find the **martiLQ** framework relevant if you:
 
-1. Have many document exchanges
+1. Have many document exchanges, such as End of Day batches
 2. Need to verify or reconcile the documents
 
 ## Data exchanges
@@ -33,8 +33,6 @@ in Mac OS X HFS.  The Microsoft NTFS supports Alternate Data Streams to achieve
 Unfortunately this information is not transferrable to other systems. 
 
 The proposition is to define a format for the sidecare file and provide common library tools that
-can be be used on multiple platforms when exchanging documents / files.
+can be be used on multiple platforms when exchanging documents / files.  Multiple documents can be 
-
+defined in a singel **martiLQ** definition which adds to efficiency and productivity if used 
-See also:
+for End of Day or similar batches - or even single file transfers.
-1. [When to use the framework](when.md)
-2. [What the framework contains](what.md)

docs/source/why.md

@@ -16,8 +16,9 @@ You would use **martiLQ** if any of the controls are a requirement for you.
 ## Documents
 
 Documents in this context are digital storage objects such as operating system files,
-cloud storage objects or blobs.  The document content has structure and contains multiple
+cloud storage objects or blobs.  The document content can have structure and contains multiple
-records.
+records or it can be unstructered such as PDFs.  If you are including PDFs then they are
+likely to be be supporting documents for your actual data files.
 
 The **martiLQ** framework is not intended to be used for single record transfers such as 
 in single web transactions.  It is for providing controls when moving large amounts of
@@ -31,7 +32,3 @@ The framework does not replace your security, inflight encryption or encryption
 You are encouraged to use TLS or SSH to connect devices and transfer documents.  Storage 
 encryption and access controls for your documents is also relevant as part of the bigger 
 picture.
-
-See also:
-1. [Who would use the framework](who.md)
-2. [When to use the framework](when.md)

martiLQ.md

@@ -1,9 +0,0 @@
-# MartiLQ document
-
-The metadata reconciliation transfer information is referred
-to as the **martiLQ** document throughout this documentation.
-
-The **martiLQ** document can be part of a message or a document
-in its own right.  If the document is a file then the recommended
-name for the document is the same name as the data file,
-including extension, with the added extension of ``.mti``

source/golang/client/src/config/batch.no

@@ -1 +1 @@
-0005.004
+0001.001

source/golang/client/src/config/martilq.ini

@@ -2,7 +2,8 @@
 [General]
 
 logPath = 
-
+tempPath = 
+dataPath = 
 
 [MartiLQ]
 
@@ -18,12 +19,13 @@ theme        = GOLANG
 
 [Resources]
 
-author   = 
+author   = Mobidick
-title    = documentName
+title    = {{documentName}}
 state    = active
 expires  = 2:0:0
 encoding = UTF-8
 version  = 
+urlPrefix = http://localhost/martilq/
 
 [Hash]
 
@@ -37,3 +39,17 @@ signKey_Password =
 proxy = 
 username =
 password = 
+
+
+[Custom_Spatial]
+
+enabled = true
+country = Netherland
+region = 
+town = Amsterdam
+
+[Custom_Temporal]
+
+enabled = true
+businessDate = {{yesterday}}
+runDate = {{today}}

source/golang/client/src/main.go

@@ -11,10 +11,15 @@ import (
 
 
 type Parameters struct {
+
+	help bool
+
 	task string
 	sourcePath string
 	recursive bool
+	urlPrefix string
 	configPath string
+	definitionPath string
 	outputPath string
 
 	title string
@@ -37,6 +42,12 @@ func loadArguments(args []string) {
 	for ix < maxArgs {
 		matched := false
 
+		if args[ix] == "-h" || args[ix] == "--help" {
+			matched = true
+			params.help = true
+			break
+		}
+
 		if args[ix] == "-t" || args[ix] == "--task" {
 			matched = true
 			if ix < maxArgs {
@@ -67,6 +78,16 @@ func loadArguments(args []string) {
 			}
 		}	
 
+		if args[ix] == "-m" ||  args[ix] == "--martilq" {
+			matched = true
+			ix = ix + 1
+			if ix < maxArgs {
+				params.definitionPath = args[ix]
+			} else {
+				panic("Missing parameter for MARTILQ")
+			}
+		}	
+
 		if args[ix] == "-o" ||  args[ix] == "--output" {
 			matched = true
 			ix = ix + 1
@@ -126,64 +147,113 @@ func loadArguments(args []string) {
 
 }
 
+func printHelp() {
+
+	fmt.Println("")
+	fmt.Println("\t   marticli_client ")
+	fmt.Println("\t   =============== ")
+	fmt.Println("")
+	fmt.Println("\tThis program is intended as a simple reference implementation")
+	fmt.Println("\tin Go of the MartiLQ framework.  It is does not provide all")
+	fmt.Println("\tthe possible functionality but enough to demonstrate the concept.")
+	fmt.Println("")
+
+	fmt.Println(" The command line arguments are:")
+	fmt.Println("")
+	fmt.Println(" -h or --help : Display this help")
+	fmt.Println(" -t or --task : Execute a predefined task which are")
+	fmt.Println("           INIT initialise a new configuration file")
+	fmt.Println("           GEN generate a MartiLQ definition file")
+	fmt.Println("           RECON reconicile a MartiLQ definition file")
+	fmt.Println(" -c or --config : Configuration file used by all tasks")
+	fmt.Println("           This is the file written by the INIT task")
+	fmt.Println(" -s or --source : Source directory or file to build MartiLQ definition")
+	fmt.Println("           This is used by the GEN and RECON task")
+	fmt.Println(" -m or --martilq : MartiLQ definition file")
+	fmt.Println("           This is used by the GEN and RECON task")
+	fmt.Println("           The GEN task generates the file while")
+	fmt.Println("           RECON task reads the file")
+	fmt.Println(" -o or --output : Output file")
+	fmt.Println("           This is used by the RECON task")
+
+	fmt.Println("")
+	fmt.Println(" --title : Title for the MartiLQ. Think of this as")
+	fmt.Println("           the job name")
+	fmt.Println("           This is used by the GEN task")
+	fmt.Println(" --description : Description for the MartiLQ. This can be text")
+	fmt.Println("           or a pointer to a file when the @ prefix is used")
+	fmt.Println("           This is used by the GEN task")
+	fmt.Println(" --landing : Landing page for the defintion in the MartiLQ")
+	fmt.Println("           This is best if it is a URL")
+	fmt.Println("           This is used by the GEN task")
+
+	fmt.Println("")
+
+}
+
 func main () {
 
 	currentDirectory, _ := os.Getwd()
 	params.sourcePath = currentDirectory
-	//params.outputPath = "" 
-	//params.configPath = "" 
 
 	loadArguments(os.Args)
 
 	matched := false
 
-	if params.task == "INIT" {
+	if params.help {
-		if params.configPath == "" {
+		printHelp()
-			panic("Missing 'config' parameter")
-		}
-
-		c := martilq.NewConfiguration()
-		if c.SaveConfig(params.configPath) != true {
-			panic("Configuration not saved to: "+ params.configPath)
-		}
-		fmt.Println("Created MARTILQ INI definition: " + params.configPath)
-		matched = true
-	}
-
-	if params.task == "GEN" {
-
-		if params.sourcePath == "" {
-			panic("Missing 'source' parameter")
-		}
-		if params.outputPath == "" {
-			panic("Missing 'output' parameter")
-		}
-
-		m := martilq.ProcessDirectory(params.configPath, params.sourcePath, params.recursive, params.outputPath )
-		if params.title != "" {
-			m.Title = params.title
-		}
-		if params.landing != "" {
-			m.LandingPage = params.landing
-		}
-		if params.description != "" {
-			m.Description = params.description
-		}
-		m.Modified = time.Now()
-		m.Save(params.outputPath)
-
-		fmt.Println("Created MARTILQ definition: " + params.outputPath)
-		matched = true
-	}
-
-	if params.task == "RECON" {
-		
-		matched = true
-	}
-
-	if matched {
-		fmt.Println("Completed " + params.task)
 	} else {
-		fmt.Println("Unknown task: " + params.task)
+
+
+		if params.task == "INIT" {
+			if params.configPath == "" {
+				panic("Missing 'config' parameter")
+			}
+
+			c := martilq.NewConfiguration()
+			if c.SaveConfig(params.configPath) != true {
+				panic("Configuration not saved to: "+ params.configPath)
+			}
+			fmt.Println("Created MARTILQ INI definition: " + params.configPath)
+			matched = true
+		}
+
+		if params.task == "GEN" {
+
+			if params.sourcePath == "" {
+				panic("Missing 'source' parameter")
+			}
+			if params.definitionPath == "" {
+				panic("Missing 'output' parameter")
+			}
+
+			m := martilq.ProcessFilePath(params.configPath, params.sourcePath, params.recursive, params.urlPrefix, params.definitionPath )
+			if params.title != "" {
+				m.Title = params.title
+			}
+			if params.landing != "" {
+				m.LandingPage = params.landing
+			}
+			if params.description != "" {
+				m.Description = params.description
+			}
+			m.Modified = time.Now()
+			m.Save(params.definitionPath)
+
+			fmt.Println("Created MARTILQ definition: " + params.definitionPath)
+			matched = true
+		}
+
+		if params.task == "RECON" {
+
+			_ = martilq.ReconcileFilePath(params.configPath, params.sourcePath, params.recursive, params.definitionPath, params.outputPath )
+			
+			matched = true
+		}
+
+		if !matched {
+			printHelp()
+		}
+
 	}
 }

source/golang/client/src/martilq/attributes_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 	"strconv"
 	"time"
-	"fmt"
 )
 
 func TestAttr_Zip(t *testing.T) {
@@ -38,7 +37,6 @@ func TestAttr_TemporalA(t *testing.T) {
 	endDate := time.Now().AddDate(0,0,-1)
 
 	a := NewDefaultTemporalAttributes(businessDate, runDate, false, startDate, endDate)
-	fmt.Println(a)
 	if len(a) != 2 {
 		t.Error("Arrays size not 2: " + strconv.Itoa(len(a)))
 	}
@@ -47,7 +45,6 @@ func TestAttr_TemporalA(t *testing.T) {
 	}
 
 	a = NewDefaultTemporalAttributes(businessDate, runDate, true, startDate, endDate)
-	fmt.Println(a)
 	if len(a) != 4 {
 		t.Error("Arrays size not 4: " + strconv.Itoa(len(a)))
 	}

source/golang/client/src/martilq/config.go

@@ -20,6 +20,8 @@ type configuration struct {
 	softwareName string
 
 	logPath string
+	tempPath string
+	dataPath string
 
 	publisher string
 	contactPoint string
@@ -33,6 +35,7 @@ type configuration struct {
 
 	title string
 	author string
+	urlPrefix string
 	state string
 	version string
 	expires string
@@ -50,6 +53,9 @@ type configuration struct {
 
 	loaded bool
 	configPath string
+
+	temporal oTemporal
+	spatial oSpatial
 }
 
 
@@ -63,7 +69,7 @@ func NewConfiguration() configuration {
 
 	c.softwareName = GetSoftwareName()
 
-	c.title = "documentName"
+	c.title = "{{documentName}}"
 	c.state = "active"
 	c.accessLevel = "Confidential"
 	c.rights = "Restricted"
@@ -71,10 +77,14 @@ func NewConfiguration() configuration {
 	c.encoding = cEncoding
 	c.batchInc = 0.001
 
+	c.urlPrefix = "file://"
 	c.hash = true
 	c.hashAlgorithm = "SHA256" 
 	c.loaded = false
 
+	c.spatial = GetSpatial()
+	c.temporal = GetTemporal()
+
 	configPath := findIni()
 	if configPath != "" {
 		c.LoadConfig(configPath)
@@ -130,6 +140,8 @@ func (c *configuration) SaveConfig(ConfigPath string) bool {
 	cfgini, _ := ini.LooseLoad("./martilq.ini")
 		
 	cfgini.Section("General").Key("logPath").SetValue (c.logPath)
+	cfgini.Section("General").Key("tempPath").SetValue (c.tempPath)
+	cfgini.Section("General").Key("dataPath").SetValue (c.dataPath)
 
 	cfgini.Section("MartiLQ").Key("tags").SetValue(c.tags)
 	cfgini.Section("MartiLQ").Key("publisher").SetValue(c.publisher)
@@ -146,6 +158,7 @@ func (c *configuration) SaveConfig(ConfigPath string) bool {
 	cfgini.Section("Resources").Key("expires").SetValue (c.expires)
 	cfgini.Section("Resources").Key("encoding").SetValue (c.encoding)
 	cfgini.Section("Resources").Key("version").SetValue (c.version)
+	cfgini.Section("Resources").Key("urlPrefix").SetValue (c.urlPrefix)
 	
 	cfgini.Section("Hash").Key("hashAlgorithm").SetValue (c.hashAlgorithm)
 	cfgini.Section("Hash").Key("signKey_File").SetValue (c.signKey_File)
@@ -161,6 +174,13 @@ func (c *configuration) SaveConfig(ConfigPath string) bool {
 		WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath))
 		return false
 	}
+
+	res := c.spatial.SaveSpatial(ConfigPath)
+	res = c.temporal.SaveTemporal(ConfigPath)
+	if res {
+
+	}
+
 	return true
 }
 
@@ -191,6 +211,8 @@ func (c *configuration) LoadConfig(ConfigPath string) bool {
 		}
 		
 		c.logPath = cfgini.Section("General").Key("logPath").MustString(c.logPath)
+		c.tempPath = cfgini.Section("General").Key("tempPath").MustString(c.tempPath)
+		c.dataPath = cfgini.Section("General").Key("dataPath").MustString(c.dataPath)
 
 		c.tags = cfgini.Section("MartiLQ").Key("tags").MustString(c.tags)
 		c.accessLevel = cfgini.Section("MartiLQ").Key("accessLevel").MustString(c.accessLevel)
@@ -201,11 +223,12 @@ func (c *configuration) LoadConfig(ConfigPath string) bool {
 		c.contactPoint = cfgini.Section("MartiLQ").Key("contactPoint").MustString(c.contactPoint)
 		c.theme= cfgini.Section("MartiLQ").Key("theme").MustString(c.theme)
 
-		c.author = cfgini.Section("Resources").Key("title").MustString(c.title)
+		c.title = cfgini.Section("Resources").Key("title").MustString(c.title)
 		c.author = cfgini.Section("Resources").Key("author").MustString(c.author)
 		c.state = cfgini.Section("Resources").Key("state").MustString(c.state)
 		c.expires = cfgini.Section("Resources").Key("expires").MustString(c.expires)
 		c.encoding = cfgini.Section("Resources").Key("encoding").MustString(c.encoding)
+		c.urlPrefix = cfgini.Section("Resources").Key("urlPrefix").MustString(c.urlPrefix)
 		
 		c.hashAlgorithm = cfgini.Section("Hash").Key("hashAlgorithm").MustString(c.hashAlgorithm)
 		c.signKey_File = cfgini.Section("Hash").Key("signKey_File").MustString(c.signKey_File)
@@ -219,7 +242,9 @@ func (c *configuration) LoadConfig(ConfigPath string) bool {
 		c.proxyUser = cfgini.Section("Network").Key("proxyUser").MustString(c.proxyUser)
 		c.proxyCredential = cfgini.Section("Network").Key("proxyCredential").MustString(c.proxyCredential)
 	
-		WriteLog(fmt.Sprintf("Loaded config file: %v", ConfigPath))
+		c.spatial, _ = LoadSpatial(ConfigPath)
+		c.temporal, _ = LoadTemporal(ConfigPath)
+
 		c.configPath = ConfigPath
 	}
 

source/golang/client/src/martilq/custom.go

@@ -2,6 +2,8 @@ package martilq
 
 import (
 	"time"
+	"fmt"
+	"gopkg.in/ini.v1"
 )
        
 type oSoftware struct {
@@ -12,16 +14,18 @@ type oSoftware struct {
 }
 
 type oTemporal struct {
+	enabled bool
 	Extension string `json:"extension"`
-	BusinessDate time.Time
+	BusinessDate time.Time  `json:"businessDate"`
-	RunDate time.Time
+	RunDate time.Time `json:"runDate"`
 }
 
 type oSpatial struct {
+	enabled bool
 	Extension string `json:"extension"`
 	Country string `json:"country"`
-	Region string
+	Region string `json:"region"`
-	Town string
+	Town string `json:"town"`
 }
 
 func GetSoftware() oSoftware {
@@ -48,3 +52,114 @@ func GetSpatial() oSpatial {
 
 	return o
 }
+
+
+func LoadSpatial(ConfigPath string) (oSpatial, error) {
+	o := oSpatial {}
+	o.Extension = "spatial"
+
+	cfgini, err := ini.Load(ConfigPath)
+	if err != nil {
+		WriteLog(fmt.Sprintf("Fail to read file: %v", ConfigPath))
+		fmt.Printf("Fail to read file: %v", err)
+		return o, err
+	}
+	
+	o.enabled = cfgini.Section("Custom_Spatial").Key("enabled").MustBool(o.enabled)
+	o.Country = cfgini.Section("Custom_Spatial").Key("country").MustString(o.Country)
+	o.Region = cfgini.Section("Custom_Spatial").Key("region").MustString(o.Region)
+	o.Town = cfgini.Section("Custom_Spatial").Key("town").MustString(o.Town)
+
+	return o, nil
+}
+
+func (s *oSpatial) SaveSpatial(ConfigPath string) bool {
+
+	cfgini, _ := ini.Load(ConfigPath)
+		
+	cfgini.Section("Custom_Spatial").Key("enabled").SetValue("false")
+	cfgini.Section("Custom_Spatial").Key("country").SetValue(s.Country)
+	cfgini.Section("Custom_Spatial").Key("region").SetValue(s.Region)
+	cfgini.Section("Custom_Spatial").Key("town").SetValue(s.Town)
+
+	err := cfgini.SaveTo(ConfigPath)
+	if err != nil {
+		WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath))
+		return false
+	}
+
+	return true
+
+}
+
+
+func LoadTemporal(ConfigPath string) (oTemporal, error) {
+	o := oTemporal {}
+	o.Extension = "temporal"
+
+	cfgini, err := ini.Load(ConfigPath)
+	if err != nil {
+		WriteLog(fmt.Sprintf("Fail to read file: %v", ConfigPath))
+		fmt.Printf("Fail to read file: %v", err)
+		return o, err
+	}
+	
+	o.enabled = cfgini.Section("Custom_Temporal").Key("enabled").MustBool(o.enabled)
+	t := cfgini.Section("Custom_Temporal").Key("businessDate").MustString("")
+	if t != "" {
+		matched := false
+		if t == "{{now}}" {
+			matched = true
+			o.BusinessDate = time.Now()
+		} 
+		if t == "{{yesterday}}" {
+			matched = true
+			o.BusinessDate = time.Now().AddDate(0,0,-1)
+			o.BusinessDate = time.Date(o.BusinessDate.Year(), o.BusinessDate.Month(), o.BusinessDate.Day(), 0, 0, 0, 0, time.Local)
+		} 
+		if t == "{{today}}" {
+			matched = true
+			o.BusinessDate = time.Now()
+			o.BusinessDate = time.Date(o.BusinessDate.Year(), o.BusinessDate.Month(), o.BusinessDate.Day(), 0, 0, 0, 0, time.Local)
+		} 
+		if !matched {
+
+		}
+	}
+	t = cfgini.Section("Custom_Temporal").Key("runDate").MustString("")
+	if t != "" {
+		matched := false
+		if t == "{{now}}" {
+			matched = true
+			o.RunDate = time.Now()
+		} 
+		if t == "{{today}}" {
+			matched = true
+			o.RunDate = time.Now()
+			o.RunDate = time.Date(o.RunDate.Year(), o.RunDate.Month(), o.RunDate.Day(), 0, 0, 0, 0, time.Local)
+		} 
+		if !matched {
+
+		}
+	}
+
+	return o, nil
+}
+
+func (s *oTemporal) SaveTemporal(ConfigPath string) bool {
+
+
+	cfgini, _ := ini.Load(ConfigPath)
+		
+	cfgini.Section("Custom_Temporal").Key("enabled").SetValue("false")
+	cfgini.Section("Custom_Temporal").Key("businessDate").SetValue("{{today}}")
+	cfgini.Section("Custom_Temporal").Key("runDate").SetValue("{{today}}")
+
+	err := cfgini.SaveTo(ConfigPath)
+	if err != nil {
+		WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath))
+		return false
+	}
+
+	return true
+}

source/golang/client/src/martilq/marti.go

@@ -55,10 +55,6 @@ func NewMarti() Marti {
 
 	software := GetSoftware()
 	m.Custom = append(m.Custom, software)
-	spatial := GetSpatial()
-	m.Custom = append(m.Custom, spatial)
-	temporal := GetTemporal()
-	m.Custom = append(m.Custom, temporal)
 
 	m.config = NewConfiguration()
 
@@ -103,6 +99,16 @@ func (m *Marti) LoadConfig(ConfigPath string) {
 	if m.config.batch != "" {
 		if m.config.batch[0] == '@' {
 			_, err := os.Stat(m.config.batch[1:])
+			if os.IsNotExist(err) {
+				// See if we can locate it in Config INI directory
+				_, fileb := filepath.Split(m.config.batch[1:])
+				dirc, _ := filepath.Split(ConfigPath)
+				_, err := os.Stat(dirc + fileb)
+				if err == nil {
+					m.config.batch = "@" + dirc + fileb
+				}
+			}
+			_, err = os.Stat(m.config.batch[1:])
 			if os.IsNotExist(err) {
 				WriteLog(fmt.Sprintf("Batch file '%v' does not exist" , m.config.batch))		
 			} else {
@@ -122,6 +128,15 @@ func (m *Marti) LoadConfig(ConfigPath string) {
 		}
 	}
 
+	if m.config.spatial.enabled == true {
+		spatial := m.config.spatial
+		m.Custom = append(m.Custom, spatial)
+	}
+	if m.config.temporal.enabled == true {
+		temporal := m.config.temporal
+		m.Custom = append(m.Custom, temporal)
+	}
+
 }
 
 
@@ -169,40 +184,74 @@ func (m *Marti) Save(pathFile string) bool {
 }
 
 
-func ProcessDirectory(ConfigPath string, SourcePath string, Recursive bool, TargetPath string) Marti {
+func ProcessFilePath(ConfigPath string, SourcePath string, Recursive bool, UrlPrefix string, DefinitionPath string) Marti {
 
 	m := NewMarti()
 
-	_, err := os.Stat(TargetPath)
+	if DefinitionPath != "" {
-	if err == nil {
+		_, err := os.Stat(DefinitionPath)
-		m, err = Load(m.config, TargetPath)
+		if err == nil {
-		if err != nil {
+			m, err = Load(m.config, DefinitionPath)
-			panic("Unable to load existing MartiLQ defintion: " + TargetPath)
+			if err != nil {
+				panic("Unable to load existing MartiLQ definition: " + DefinitionPath)
+			}
+			// Update the batch number, minor version
+			m.Batch = math.Round((m.Batch + m.config.batchInc)/m.config.batchInc)*m.config.batchInc
+			m.config.LoadConfig(ConfigPath)
+		}  else {
+			if ConfigPath != "" {
+				m.LoadConfig(ConfigPath)
+			}	
 		}
-		// Update the batch number, minor version
-		m.Batch = math.Round((m.Batch + m.config.batchInc)/m.config.batchInc)*m.config.batchInc
-		m.config.LoadConfig(ConfigPath)
 	} else {
 		if ConfigPath != "" {
 			m.LoadConfig(ConfigPath)
 		}
 	}
 
-	filepath.Walk(SourcePath, func(path string, info os.FileInfo, err error) error {
+	fileStat, err := os.Stat(SourcePath)
-        if err != nil {
+	if err != nil {
-            log.Fatalf(err.Error())
+		panic("Source path does not exist or is inaccessible: " + SourcePath)
-        }
+	} else {
-		if info.IsDir() {
-			if Recursive {
 
-			}
+		if UrlPrefix == "" {
-		} else {
+			UrlPrefix = m.config.urlPrefix
-			url := "file://"+info.Name()
-			m.AddResource(info.Name(), path, url) 
 		}
-        return nil
+		if UrlPrefix == "" {
-    })
+			UrlPrefix = "file://"
+		}
+
+		if fileStat.IsDir() {
+
+			filepath.Walk(SourcePath, func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					log.Fatalf(err.Error())
+				}
+				if info.IsDir() {
+					if Recursive {
+
+					}
+				} else {
+					url := UrlPrefix+info.Name()
+					m.AddResource(info.Name(), path, url) 
+				}
+				return nil
+			})
+		} else {
+			url := UrlPrefix+fileStat.Name()
+			m.AddResource(fileStat.Name(), SourcePath, url) 
+		}
+	}
 
 	return m
 
 }
+
+
+func ReconcileFilePath(ConfigPath string, SourcePath string, Recursive bool, DefinitionPath string, OutputPath string) Marti {
+
+	m := NewMarti()
+
+
+	return m
+}

source/golang/client/src/martilq/marti_test.go

@@ -45,8 +45,8 @@ func TestMarti_DirectoryA(t *testing.T) {
 	currentDirectory, _ := os.Getwd()
 	SourcePath := currentDirectory
 	Recursive := false
-	TargetPath := "../test/test_martilq_directoryA.json"
+	DefPath := "../test/test_martilq_directoryA.json"
-	ProcessDirectory("", SourcePath, Recursive, TargetPath) 
+	ProcessFilePath("", SourcePath, Recursive, DefPath, "") 
 
 }
 
@@ -55,7 +55,7 @@ func TestMarti_DirectoryB(t *testing.T) {
 	currentDirectory, _ := os.Getwd()
 	SourcePath := currentDirectory
 	Recursive := false
-	TargetPath := "../test/test_martilq_directoryB.json"
+	DefPath := "../test/test_martilq_directoryB.json"
-	ProcessDirectory("../config/martilq.ini", SourcePath, Recursive, TargetPath) 
+	ProcessFilePath("../config/martilq.ini", SourcePath, Recursive, DefPath, "") 
 
 }

source/golang/client/src/martilq/resource.go

@@ -11,15 +11,15 @@ import (
 
 type Resource struct {
 	Title string `json:"title"`
-	Uid string `'json:"uid"`
+	Uid string `json:"uid"`
-	DocumentName string `json:"documentName`
+	DocumentName string `json:"documentName"`
 	IssueDate time.Time `json:"issueDate"`
 	Modified time.Time `json:"modified"`
 	Expires time.Time `json:"expires"`
 	State string `json:"state"`
 	Author string `json:"author"`
 	Length int64 `json:"length"`
-	Hash hash
+	Hash hash `json:"hash"`
 
 	Description string `json:"description"`
 	Url string `json:"url"`
@@ -28,6 +28,7 @@ type Resource struct {
 	Encoding string `json:"encoding"`
 	Compression string `json:"compression"`
 	Encryption string `json:"encryption"`
+	DescribedBy string `json:"describedBy"`
 
 	Attributes []Attribute `json:"attributes"`
 }
@@ -59,6 +60,11 @@ func NewMartiLQResource(config configuration, sourcePath string, urlPath string,
 		}
 	}
 
+
+	if config.dataPath != "" {
+
+	}
+
 	u := uuid.New()
 	r.Uid = u.String()
 
@@ -68,7 +74,7 @@ func NewMartiLQResource(config configuration, sourcePath string, urlPath string,
 	r.Encoding = config.encoding
 
 	r.DocumentName = stats.Name()
-	if config.title == "documentName" {
+	if config.title == "{{documentName}}" {
 		r.Title = r.DocumentName
 	}
 	r.IssueDate = time.Now()