parent
5859636488
commit
4284922908
|
|
@ -11,9 +11,13 @@ buildNumber.properties
|
|||
.mvn/wrapper/maven-wrapper.jar
|
||||
|
||||
test/powershell/results/
|
||||
docs/samples/powershell/test/
|
||||
|
||||
docs/samples/python/test
|
||||
docs/source/samples/powershell/test/
|
||||
docs/source/samples/python/test
|
||||
|
||||
docs/build
|
||||
|
||||
source/python/client/__pycache__
|
||||
|
||||
source/marti/source/golang/client/src/test
|
||||
source/golang/client/src/test
|
||||
|
||||
|
|
|
|||
16
README.md
16
README.md
|
|
@ -40,7 +40,7 @@ information
|
|||
## Transfer information
|
||||
|
||||
The information in the **martiLQ** document is summarised below. For more detailed
|
||||
information see [martiLQ definition](martiLQ.md)
|
||||
information see [martiLQ definition](/docs/source/martiLQ.md)
|
||||
|
||||
### Mandatory information
|
||||
|
||||
|
|
@ -48,7 +48,7 @@ The mandatory information is:
|
|||
|
||||
* Title
|
||||
* Unique identifier
|
||||
* Distribution list - See Distribution section summary below or detailed document [Distribution](docs/distribution.md)
|
||||
* Resource list - See Resource section summary below or detailed document [Resource](docs/source/resource.md)
|
||||
|
||||
### Optional information
|
||||
|
||||
|
|
@ -75,11 +75,11 @@ The optional information is:
|
|||
The information supplied can be extended by party agreement and there
|
||||
are place holders in the defintion.
|
||||
|
||||
### Distribution
|
||||
### Resource
|
||||
|
||||
The distribution section is intended to allow multiple data files to be
|
||||
grouped together. The distribution section can be repeated, but at least
|
||||
one must be included. If the distribution is repeated it will commonly
|
||||
The resource section is intended to allow multiple data files to be
|
||||
grouped together. The resource section can be repeated, but at least
|
||||
one must be included. If the resource is repeated it will commonly
|
||||
be for definiting multiple formats of the same data or batching of
|
||||
different data together from the same extract process.
|
||||
|
||||
|
|
@ -92,9 +92,9 @@ different data together from the same extract process.
|
|||
* Hash of document - The hash of the document, which can be blank especially for large documents
|
||||
* Hash algorithm
|
||||
|
||||
### Distribution optional
|
||||
### Resource optional
|
||||
|
||||
The following are some of the optional items in the distribution section. See [Distribution](docs/distribution.md)
|
||||
The following are some of the optional items in the resource section. See [Resource](docs/source/resources.md)
|
||||
for more details
|
||||
|
||||
* Description
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line, and also
|
||||
# from the environment for the first two.
|
||||
SPHINXOPTS ?=
|
||||
SPHINXBUILD ?= sphinx-build
|
||||
SOURCEDIR = source
|
||||
BUILDDIR = build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
|
||||
|
||||
https://ckan.org/
|
||||
|
||||
Sample Json from https://data.gov.au/data/dataset/f2b7c2c1-f4ef-4ae9-aba5-45c19e4d3038
|
||||
|
|
@ -1 +0,0 @@
|
|||
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
@ECHO OFF
|
||||
|
||||
pushd %~dp0
|
||||
|
||||
REM Command file for Sphinx documentation
|
||||
|
||||
if "%SPHINXBUILD%" == "" (
|
||||
set SPHINXBUILD=sphinx-build
|
||||
)
|
||||
set SOURCEDIR=source
|
||||
set BUILDDIR=build
|
||||
|
||||
if "%1" == "" goto help
|
||||
|
||||
%SPHINXBUILD% >NUL 2>NUL
|
||||
if errorlevel 9009 (
|
||||
echo.
|
||||
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
|
||||
echo.installed, then set the SPHINXBUILD environment variable to point
|
||||
echo.to the full path of the 'sphinx-build' executable. Alternatively you
|
||||
echo.may add the Sphinx directory to PATH.
|
||||
echo.
|
||||
echo.If you don't have Sphinx installed, grab it from
|
||||
echo.https://www.sphinx-doc.org/
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
goto end
|
||||
|
||||
:help
|
||||
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
|
||||
:end
|
||||
popd
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
# Quality definition
|
||||
|
||||
The **martiLQ** definition allows for the inclusion of a load quality
|
||||
definition. This load quality definition is intended to be
|
||||
able to be applied universally with common tools. As such not
|
||||
all needs are covered.
|
||||
|
||||
## Defined laod quality metrics
|
||||
|
||||
|
||||
* Number of records in the document - This is the number of data primary records not the
|
||||
count of end of lines and is agreed between parties. XML record counts could be based
|
||||
on the number of primary segments under root. JSON records can be counted in a similar way.
|
||||
The headers or trailling records are not counted
|
||||
|
||||
|
||||
Sequence number - linked to the job producing the document and therefore a daily, weekly and monthly extracts for the
|
||||
same document would have different sequence numbers
|
||||
|
||||
Discourage us of magic formats for document names such as
|
||||
|
||||
XT_PARTY_20210911_SQ00001_N000567891234_V01.DAT
|
||||
|
||||
Header and Trailer records are not part of the quality definition except
|
||||
|
||||
In fact the trailer record is intended to be replaced by this quality definition
|
||||
|
||||
The header, if it exists, is only used where it identifies the column name sequence of the data.
|
||||
|
||||
Effective and Process date -----------------
|
||||
|
||||
Row count -
|
||||
Column count -
|
||||
Depth count -
|
||||
|
||||
Mandatory - data must be present and cannot be blank or null
|
||||
Uniqueness - data value must be unique within the document
|
||||
Data integrity - data exists within defined tolerances
|
||||
|
||||
|
||||
row count 9999
|
||||
|
||||
column count 9999
|
||||
|
||||
column_name sum 9999
|
||||
|
||||
column_name gt 9999
|
||||
column_name lt 9999
|
||||
column_name eq 9999
|
||||
column_name eq " "
|
||||
column_name ne 9999
|
||||
column_name ne " " == Check for value
|
||||
column_name ge 9999
|
||||
column_name le " "
|
||||
column_name in " ", " ", " "
|
||||
|
||||
column_name is integer
|
||||
column_name is decimal
|
||||
column_name is unique
|
||||
|
||||
|
|
@ -21,4 +21,4 @@ can adjust if they resonate with your circumstances,
|
|||
6. [Comparison of martiLQ definition](comparison.md)
|
||||
7. [References](references.md)
|
||||
|
||||
[!INCLUDE [martiLQ High Level Definition](../martiLQ.md)]
|
||||
[!INCLUDE [martiLQ High Level Definition](martiLQ.md)]
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
# Attribute definition
|
||||
|
||||
A Resource can list attributes related to the document / file.
|
||||
|
||||
An attribute is a generic definition and conventions are
|
||||
observed in the definitions that are captured here.
|
||||
|
||||
## Attribute definition
|
||||
|
||||
The Attribute consists of:
|
||||
|
||||
* category - A value of "dataset",
|
||||
* name
|
||||
* function - A value such as "count"
|
||||
* comparison - A comparisn value or NA. Values are "EQ",
|
||||
"NE", "GT", "LT
|
||||
* value - The value for the attribute based on the above complex key, excluding comparison
|
||||
|
||||
A sample JSON is shown below which describes the
|
||||
number of records in the file for the given format.
|
||||
|
||||
```json
|
||||
"attributes": [
|
||||
{
|
||||
"category": "dataset",
|
||||
"name": "records",
|
||||
"function": "count",
|
||||
"comparison": "EQ",
|
||||
"value": "9"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
# CKAN definition
|
||||
|
||||
The **martiLQ** has used similar terms and structures that are found in the
|
||||
CKAN API describing the resources. This similarity allows for simple mapping of values
|
||||
from the CKAN format to **martiLQ** format.
|
||||
|
||||
What **martiLQ** brings above the CKAN definition is attributes that can be
|
||||
used to reconcile with the recieved data, plus the ability to define
|
||||
compressed and encrypted resources.
|
||||
|
||||
For more information on CKAN see https://ckan.org/
|
||||
|
||||
A sample Json to compare against the **martiLQ** definition
|
||||
is https://data.gov.au/data/dataset/f2b7c2c1-f4ef-4ae9-aba5-45c19e4d3038
|
||||
|
||||
|
|
@ -34,10 +34,10 @@ Samples exist on CKAN integration.
|
|||
## Magda and martiLQ
|
||||
|
||||
Another source of data is [Magda](https://magda.io/) which has API metadata
|
||||
definitions. Magda is more about data fedaration and as such provides
|
||||
definitions. Magda is more about data federation and as such provides
|
||||
functionality on finding data sources and describing the contents.
|
||||
|
||||
The Magda software is able to generate APIs and data content. This does not
|
||||
address the needs of data processing pipeline when reconciliation is required.
|
||||
|
||||
If you have Magda data sources then synergies exist between Magda and marti.
|
||||
If you have Magda data sources then synergies exist between Magda and martiLQ.
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
# Configuration file for the Sphinx documentation builder.
|
||||
#
|
||||
# This file only contains a selection of the most common options. For a full
|
||||
# list see the documentation:
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
||||
|
||||
# -- Path setup --------------------------------------------------------------
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#
|
||||
# import os
|
||||
# import sys
|
||||
# sys.path.insert(0, os.path.abspath('.'))
|
||||
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
|
||||
project = 'MartiLQ'
|
||||
copyright = '2021, meerkat@merebox.coom'
|
||||
author = 'meerkat@merebox.coom'
|
||||
|
||||
# The full version, including alpha/beta/rc tags
|
||||
release = '0.0.1'
|
||||
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = ['myst_parser']
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This pattern also affects html_static_path and html_extra_path.
|
||||
exclude_patterns = [
|
||||
'samples/python/test/*',
|
||||
'samples/powershell/test/*',
|
||||
]
|
||||
|
||||
source_suffix = {
|
||||
'.rst': 'restructuredtext',
|
||||
'.txt': 'restructuredtext',
|
||||
'.md': 'markdown',
|
||||
}
|
||||
|
||||
# -- Options for HTML output -------------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#
|
||||
#html_theme = 'alabaster'
|
||||
html_theme = 'sphinx_rtd_theme'
|
||||
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
# Custom Definition
|
||||
|
||||
The custome definition section allows the inclusion of extensions
|
||||
to the standard. To demonstrate the inclusion, there are three
|
||||
sample extensions. These are:
|
||||
|
||||
* Software - describing the **martiLQ** software version
|
||||
* Spatial - Defining the geographical boundary of the documents
|
||||
* Temporal - Defining date and time aspects of the documents
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
.. MartiLQ documentation master file, created by
|
||||
sphinx-quickstart on Sun Oct 31 14:30:06 2021.
|
||||
You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
|
||||
Welcome to MartiLQ's documentation!
|
||||
===================================
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
:maxdepth: 2
|
||||
|
||||
objective.md
|
||||
martiLQ.md
|
||||
why.md
|
||||
what.md
|
||||
when.md
|
||||
who.md
|
||||
resources.md
|
||||
attributes.md
|
||||
custom.md
|
||||
quality.md
|
||||
comparison.md
|
||||
ckan.md
|
||||
magda.md
|
||||
references.md
|
||||
|
||||
**martiLQ** stands for metadata reconcilation for transfer information, load quality.
|
||||
|
||||
Before starting with **martiLQ** it is advisable to understand if it is right for
|
||||
your organisation's needs. Information is available in a number of short
|
||||
documents. Start with the (objectives)[objective.md]
|
||||
|
||||
There are sample implementations which you
|
||||
can adjust if they resonate with your circumstances.
|
||||
|
||||
The source, documentation and samples are available at `<https://github.com/meerkat-manor/marti>`_
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
|
||||
# Magda definitions
|
||||
|
||||
https://magda.io/
|
||||
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
# MartiLQ document
|
||||
|
||||
The metadata reconciliation transfer information is referred
|
||||
to as the **martiLQ** document throughout this documentation.
|
||||
|
||||
The **martiLQ** document can be part of a message or a file
|
||||
in its own right. The definition is curently a Json file.
|
||||
|
||||
|
||||
|
||||
### Mandatory information
|
||||
|
||||
The mandatory information is:
|
||||
|
||||
* Title
|
||||
* Unique identifier
|
||||
* Resource list - See Resource section summary below or detailed document [Resource](resources.md)
|
||||
|
||||
### Optional information
|
||||
|
||||
The optional information is:
|
||||
|
||||
* Description
|
||||
* Modified
|
||||
* Tags or keywords
|
||||
* Publisher
|
||||
* Contact point
|
||||
* Acces level
|
||||
* Rights
|
||||
* Batch
|
||||
* License
|
||||
* Described By - A link to the metadata describing the document.
|
||||
More detailed information could be supplied at the link
|
||||
* Landing page
|
||||
* Theme
|
||||
* Custom list - List of custom entries, one being the **martiLQ** software details
|
||||
see [custom](custom.md)
|
||||
|
||||
### Information extension
|
||||
|
||||
The information supplied can be extended by party agreement and there
|
||||
are place holders in the defintion.
|
||||
|
||||
### Resource
|
||||
|
||||
The resource section is a list of documents or files that are to be grouped
|
||||
together are listed under the same **martiLQ** definition.
|
||||
|
||||
At least one document or file must be included. If the same resource is repeated
|
||||
it will commonly be for definiting multiple formats, with each file having a
|
||||
different extension. Commonly the definition includes at least the following
|
||||
items:
|
||||
|
||||
* Title - A summary description of the document contents
|
||||
* Unique identifier - A unique identifier, commonly a GUID
|
||||
* Document name - A name of the document such as thefile name
|
||||
* Issued date - When the document was made available. The date can include time
|
||||
* Modified - When the document was created or modified. This is the data and time
|
||||
* Size of document - The document size in bytes
|
||||
* URL - This can be ``file://``, ``https://``, ``ftp://``, etc resource location
|
||||
|
||||
### Resource optional
|
||||
|
||||
The following are some of the optional items in the resource section. See [Resource](resources.md)
|
||||
for more details
|
||||
|
||||
* Hash of document - The hash of the document, which can be blank especially for large documents
|
||||
* Hash algorithm - Algoroithm used to generate the hash value or sign it
|
||||
* Description - A more detailed description
|
||||
* Version - A document version
|
||||
* Encoding
|
||||
* Content Type
|
||||
* Compression
|
||||
* Encryption
|
||||
* Author
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# MartiLQ objective
|
||||
|
||||
The objective of **martLQ** is to define a simle standard for
|
||||
capturing the data files being transferred. It is not for
|
||||
real time web service transactions.
|
||||
|
||||
**martiLQ** is about file and document transfer and reconciling
|
||||
that the all files have arrived and have not changed, and if so
|
||||
required are also encrypted.
|
||||
|
||||
The proposition is to have a common, machine readable format
|
||||
for file exchange that:
|
||||
|
||||
* ensures data load quality and reconciles
|
||||
* can be used on Linux or Windows or Mac
|
||||
* can be used with Python, Java, PowerShell, Golang, etc
|
||||
* can be used by web services
|
||||
* uses a text based format (JSON)
|
||||
* can form part of the data processing pipeline
|
||||
|
||||
And finally is easy to understand.
|
||||
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
# Quality definition
|
||||
|
||||
The **martiLQ** definition allows for the inclusion of load quality
|
||||
definitions. The load quality definition is intended to be
|
||||
able to be applied universally with common tools. Not
|
||||
all needs are covered with the base definition but can be extended.
|
||||
|
||||
## Defined load quality metrics
|
||||
|
||||
* Sequential batch number - This is a decimal number defined at the **martiLQ** definition
|
||||
and applies to all resources. The integer portion is for new batches and the fraction
|
||||
part can be used for issues with the same data extract. such as requiring resend because
|
||||
a resource was missing.
|
||||
|
||||
* Number of records in the document - This is the number of data primary records not the
|
||||
count of end of lines and is agreed between parties. XML record counts could be based
|
||||
on the number of primary segments under root. JSON records can be counted in a similar way.
|
||||
The headers or trailling records are not counted
|
||||
|
||||
## Addresses deficiencies
|
||||
|
||||
The **martiLQ** objective is to address deficiencies with alternative
|
||||
data load quality approaches such as:
|
||||
|
||||
* magic formats in file names
|
||||
* identifying the number of files
|
||||
* knowing when all files are ready
|
||||
* separate documentation that is unlinked
|
||||
* securing the data
|
||||
* adding footers to the data, requiring custom file handlers
|
||||
|
||||
## Extending load quality metrics
|
||||
|
||||
**martLQ** framewokr and importantly is open to extension so that extra
|
||||
load metrics appropriate to the situation can be included.
|
||||
|
||||
### Extension ideas
|
||||
|
||||
The following extensions for load quality can easily be included:
|
||||
|
||||
* Mandatory data in column
|
||||
* Uniqueness of data values
|
||||
* Data values are within defined tolerances
|
||||
* Check for data exclusions
|
||||
|
||||
And all this information is included in the **martiLQ** definition
|
||||
allowing for self describing load quality.
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
|
||||
# References
|
||||
|
||||
|
||||
https://dex.dss.gov.au/sites/default/files/documents/2021-06/data-exchange-protocols-june-2021.pdf
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# Distribution definition
|
||||
# Resources definition
|
||||
|
||||
The distrubution section defines the files that are grouped
|
||||
The resources section defines the files that are grouped
|
||||
together by association. This association is not defined but can
|
||||
include different formats of the same data or a common batch extract
|
||||
such as end of day.
|
||||
|
|
@ -10,7 +10,7 @@ compressed with a utility such as WinZIP or 7ZIP. In the situation
|
|||
where a ZIP file expands to multiple documents, then the expectation is
|
||||
that the ZIP file contains a **martiLQ** document describing its contents.
|
||||
|
||||
The elements in the distribution section are:
|
||||
The elements in the resource section are:
|
||||
|
||||
* Title
|
||||
* Document name - Commonly being absolute or relative file name.
|
||||
|
|
@ -20,8 +20,9 @@ The elements in the distribution section are:
|
|||
* Size of file - The file size in bytes
|
||||
* Hash of file - The hash of the file, which can be blank especially for large files
|
||||
* Hash algorithm
|
||||
* Attributes - List of attributes associated with the document
|
||||
|
||||
The following are optional in the distribution section.
|
||||
The following are optional in the resource section.
|
||||
|
||||
* Identifier
|
||||
* Description
|
||||
|
|
@ -29,8 +30,7 @@ The following are optional in the distribution section.
|
|||
* Version - File version. The same file could be updated or this might denote the next version
|
||||
of a regular report. For example a daily extract will have the version number incremented
|
||||
every day and provide a new URL. The previous file can be retained.
|
||||
* Format - if not specified then the consumer will in all likelihood use the file extension / mime type
|
||||
* Media Type
|
||||
* Content type - if not specified then the consumer will in all likelihood use the file extension / mime type
|
||||
* Expiry Date - The date and time that this file expires and can be removed from the download URL
|
||||
location. This is not the file retention period as might be required for archiving.
|
||||
* Described By - A link to the metadata describing this file data and format
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# What is marti
|
||||
|
||||
The foundation pillar for the **martiLQ** framework is the [martiLQ document](../martiLQ.md)
|
||||
The foundation pillar for the **martiLQ** framework is the [martiLQ document](martiLQ.md)
|
||||
that defines the reconciliation and other metadata of the document / file being transferred.
|
||||
|
||||
A definition, while fundamental, benefits from having tools that can create, read and
|
||||
|
|
@ -12,7 +12,7 @@ for various programming languages and situations. As many programming languages
|
|||
generate portable programs that can execute on multiple operating systems, the
|
||||
likelihood is that a tools exists for you.
|
||||
|
||||
The source for tools is provided in this repository and some have precompiled
|
||||
The source for tools is provided in the Github repository and some have precompiled
|
||||
images.
|
||||
|
||||
See the [source](../source/) directory for more details.
|
||||
See the project source directory for more details.
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
# When would you use martiLQ
|
||||
|
||||
You are likely to start using the **martiLQ** framework when:
|
||||
|
||||
1. you have no existing standard or framework or;
|
||||
2. your existing standards no longer meets you needs or;
|
||||
3. you are starting document exchange with another business or division that uses the framework or;
|
||||
|
|
@ -8,7 +9,7 @@ You are likely to start using the **martiLQ** framework when:
|
|||
|
||||
If you already have a standard and it works for you, and you have no upcoming (large)
|
||||
initiative that would benefit from the framework, then stick with what you have. The benefits
|
||||
of the framework are unlikely to weigh in your framework
|
||||
of the framework are unlikely to weigh in your framework.
|
||||
|
||||
## Read the material
|
||||
|
||||
|
|
@ -19,6 +20,3 @@ your direction will more than likely add to your support costs.
|
|||
**Note**: As the framework is based on open structure and can be processed independently
|
||||
without the tools, you can export the definitions to other tools / future methods.
|
||||
|
||||
See also:
|
||||
1. [What the framework contains](what.md)
|
||||
2. [Quality definition](quality.md)
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
# Who is likely to use marti
|
||||
# Who is likely to use martiLQ
|
||||
|
||||
You are likely to find the **martiLQ** framework relevant if you:
|
||||
|
||||
1. Have many document exchanges
|
||||
1. Have many document exchanges, such as End of Day batches
|
||||
2. Need to verify or reconcile the documents
|
||||
|
||||
## Data exchanges
|
||||
|
|
@ -33,8 +33,6 @@ in Mac OS X HFS. The Microsoft NTFS supports Alternate Data Streams to achieve
|
|||
Unfortunately this information is not transferrable to other systems.
|
||||
|
||||
The proposition is to define a format for the sidecare file and provide common library tools that
|
||||
can be be used on multiple platforms when exchanging documents / files.
|
||||
|
||||
See also:
|
||||
1. [When to use the framework](when.md)
|
||||
2. [What the framework contains](what.md)
|
||||
can be be used on multiple platforms when exchanging documents / files. Multiple documents can be
|
||||
defined in a singel **martiLQ** definition which adds to efficiency and productivity if used
|
||||
for End of Day or similar batches - or even single file transfers.
|
||||
|
|
@ -16,8 +16,9 @@ You would use **martiLQ** if any of the controls are a requirement for you.
|
|||
## Documents
|
||||
|
||||
Documents in this context are digital storage objects such as operating system files,
|
||||
cloud storage objects or blobs. The document content has structure and contains multiple
|
||||
records.
|
||||
cloud storage objects or blobs. The document content can have structure and contains multiple
|
||||
records or it can be unstructered such as PDFs. If you are including PDFs then they are
|
||||
likely to be be supporting documents for your actual data files.
|
||||
|
||||
The **martiLQ** framework is not intended to be used for single record transfers such as
|
||||
in single web transactions. It is for providing controls when moving large amounts of
|
||||
|
|
@ -31,7 +32,3 @@ The framework does not replace your security, inflight encryption or encryption
|
|||
You are encouraged to use TLS or SSH to connect devices and transfer documents. Storage
|
||||
encryption and access controls for your documents is also relevant as part of the bigger
|
||||
picture.
|
||||
|
||||
See also:
|
||||
1. [Who would use the framework](who.md)
|
||||
2. [When to use the framework](when.md)
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
# MartiLQ document
|
||||
|
||||
The metadata reconciliation transfer information is referred
|
||||
to as the **martiLQ** document throughout this documentation.
|
||||
|
||||
The **martiLQ** document can be part of a message or a document
|
||||
in its own right. If the document is a file then the recommended
|
||||
name for the document is the same name as the data file,
|
||||
including extension, with the added extension of ``.mti``
|
||||
|
|
@ -1 +1 @@
|
|||
0005.004
|
||||
0001.001
|
||||
|
|
@ -2,7 +2,8 @@
|
|||
[General]
|
||||
|
||||
logPath =
|
||||
|
||||
tempPath =
|
||||
dataPath =
|
||||
|
||||
[MartiLQ]
|
||||
|
||||
|
|
@ -18,12 +19,13 @@ theme = GOLANG
|
|||
|
||||
[Resources]
|
||||
|
||||
author =
|
||||
title = documentName
|
||||
author = Mobidick
|
||||
title = {{documentName}}
|
||||
state = active
|
||||
expires = 2:0:0
|
||||
encoding = UTF-8
|
||||
version =
|
||||
urlPrefix = http://localhost/martilq/
|
||||
|
||||
[Hash]
|
||||
|
||||
|
|
@ -37,3 +39,17 @@ signKey_Password =
|
|||
proxy =
|
||||
username =
|
||||
password =
|
||||
|
||||
|
||||
[Custom_Spatial]
|
||||
|
||||
enabled = true
|
||||
country = Netherland
|
||||
region =
|
||||
town = Amsterdam
|
||||
|
||||
[Custom_Temporal]
|
||||
|
||||
enabled = true
|
||||
businessDate = {{yesterday}}
|
||||
runDate = {{today}}
|
||||
|
|
|
|||
|
|
@ -11,10 +11,15 @@ import (
|
|||
|
||||
|
||||
type Parameters struct {
|
||||
|
||||
help bool
|
||||
|
||||
task string
|
||||
sourcePath string
|
||||
recursive bool
|
||||
urlPrefix string
|
||||
configPath string
|
||||
definitionPath string
|
||||
outputPath string
|
||||
|
||||
title string
|
||||
|
|
@ -37,6 +42,12 @@ func loadArguments(args []string) {
|
|||
for ix < maxArgs {
|
||||
matched := false
|
||||
|
||||
if args[ix] == "-h" || args[ix] == "--help" {
|
||||
matched = true
|
||||
params.help = true
|
||||
break
|
||||
}
|
||||
|
||||
if args[ix] == "-t" || args[ix] == "--task" {
|
||||
matched = true
|
||||
if ix < maxArgs {
|
||||
|
|
@ -67,6 +78,16 @@ func loadArguments(args []string) {
|
|||
}
|
||||
}
|
||||
|
||||
if args[ix] == "-m" || args[ix] == "--martilq" {
|
||||
matched = true
|
||||
ix = ix + 1
|
||||
if ix < maxArgs {
|
||||
params.definitionPath = args[ix]
|
||||
} else {
|
||||
panic("Missing parameter for MARTILQ")
|
||||
}
|
||||
}
|
||||
|
||||
if args[ix] == "-o" || args[ix] == "--output" {
|
||||
matched = true
|
||||
ix = ix + 1
|
||||
|
|
@ -126,17 +147,64 @@ func loadArguments(args []string) {
|
|||
|
||||
}
|
||||
|
||||
func printHelp() {
|
||||
|
||||
fmt.Println("")
|
||||
fmt.Println("\t marticli_client ")
|
||||
fmt.Println("\t =============== ")
|
||||
fmt.Println("")
|
||||
fmt.Println("\tThis program is intended as a simple reference implementation")
|
||||
fmt.Println("\tin Go of the MartiLQ framework. It is does not provide all")
|
||||
fmt.Println("\tthe possible functionality but enough to demonstrate the concept.")
|
||||
fmt.Println("")
|
||||
|
||||
fmt.Println(" The command line arguments are:")
|
||||
fmt.Println("")
|
||||
fmt.Println(" -h or --help : Display this help")
|
||||
fmt.Println(" -t or --task : Execute a predefined task which are")
|
||||
fmt.Println(" INIT initialise a new configuration file")
|
||||
fmt.Println(" GEN generate a MartiLQ definition file")
|
||||
fmt.Println(" RECON reconicile a MartiLQ definition file")
|
||||
fmt.Println(" -c or --config : Configuration file used by all tasks")
|
||||
fmt.Println(" This is the file written by the INIT task")
|
||||
fmt.Println(" -s or --source : Source directory or file to build MartiLQ definition")
|
||||
fmt.Println(" This is used by the GEN and RECON task")
|
||||
fmt.Println(" -m or --martilq : MartiLQ definition file")
|
||||
fmt.Println(" This is used by the GEN and RECON task")
|
||||
fmt.Println(" The GEN task generates the file while")
|
||||
fmt.Println(" RECON task reads the file")
|
||||
fmt.Println(" -o or --output : Output file")
|
||||
fmt.Println(" This is used by the RECON task")
|
||||
|
||||
fmt.Println("")
|
||||
fmt.Println(" --title : Title for the MartiLQ. Think of this as")
|
||||
fmt.Println(" the job name")
|
||||
fmt.Println(" This is used by the GEN task")
|
||||
fmt.Println(" --description : Description for the MartiLQ. This can be text")
|
||||
fmt.Println(" or a pointer to a file when the @ prefix is used")
|
||||
fmt.Println(" This is used by the GEN task")
|
||||
fmt.Println(" --landing : Landing page for the defintion in the MartiLQ")
|
||||
fmt.Println(" This is best if it is a URL")
|
||||
fmt.Println(" This is used by the GEN task")
|
||||
|
||||
fmt.Println("")
|
||||
|
||||
}
|
||||
|
||||
func main () {
|
||||
|
||||
currentDirectory, _ := os.Getwd()
|
||||
params.sourcePath = currentDirectory
|
||||
//params.outputPath = ""
|
||||
//params.configPath = ""
|
||||
|
||||
loadArguments(os.Args)
|
||||
|
||||
matched := false
|
||||
|
||||
if params.help {
|
||||
printHelp()
|
||||
} else {
|
||||
|
||||
|
||||
if params.task == "INIT" {
|
||||
if params.configPath == "" {
|
||||
panic("Missing 'config' parameter")
|
||||
|
|
@ -155,11 +223,11 @@ func main () {
|
|||
if params.sourcePath == "" {
|
||||
panic("Missing 'source' parameter")
|
||||
}
|
||||
if params.outputPath == "" {
|
||||
if params.definitionPath == "" {
|
||||
panic("Missing 'output' parameter")
|
||||
}
|
||||
|
||||
m := martilq.ProcessDirectory(params.configPath, params.sourcePath, params.recursive, params.outputPath )
|
||||
m := martilq.ProcessFilePath(params.configPath, params.sourcePath, params.recursive, params.urlPrefix, params.definitionPath )
|
||||
if params.title != "" {
|
||||
m.Title = params.title
|
||||
}
|
||||
|
|
@ -170,20 +238,22 @@ func main () {
|
|||
m.Description = params.description
|
||||
}
|
||||
m.Modified = time.Now()
|
||||
m.Save(params.outputPath)
|
||||
m.Save(params.definitionPath)
|
||||
|
||||
fmt.Println("Created MARTILQ definition: " + params.outputPath)
|
||||
fmt.Println("Created MARTILQ definition: " + params.definitionPath)
|
||||
matched = true
|
||||
}
|
||||
|
||||
if params.task == "RECON" {
|
||||
|
||||
_ = martilq.ReconcileFilePath(params.configPath, params.sourcePath, params.recursive, params.definitionPath, params.outputPath )
|
||||
|
||||
matched = true
|
||||
}
|
||||
|
||||
if matched {
|
||||
fmt.Println("Completed " + params.task)
|
||||
} else {
|
||||
fmt.Println("Unknown task: " + params.task)
|
||||
if !matched {
|
||||
printHelp()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import (
|
|||
"testing"
|
||||
"strconv"
|
||||
"time"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func TestAttr_Zip(t *testing.T) {
|
||||
|
|
@ -38,7 +37,6 @@ func TestAttr_TemporalA(t *testing.T) {
|
|||
endDate := time.Now().AddDate(0,0,-1)
|
||||
|
||||
a := NewDefaultTemporalAttributes(businessDate, runDate, false, startDate, endDate)
|
||||
fmt.Println(a)
|
||||
if len(a) != 2 {
|
||||
t.Error("Arrays size not 2: " + strconv.Itoa(len(a)))
|
||||
}
|
||||
|
|
@ -47,7 +45,6 @@ func TestAttr_TemporalA(t *testing.T) {
|
|||
}
|
||||
|
||||
a = NewDefaultTemporalAttributes(businessDate, runDate, true, startDate, endDate)
|
||||
fmt.Println(a)
|
||||
if len(a) != 4 {
|
||||
t.Error("Arrays size not 4: " + strconv.Itoa(len(a)))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ type configuration struct {
|
|||
softwareName string
|
||||
|
||||
logPath string
|
||||
tempPath string
|
||||
dataPath string
|
||||
|
||||
publisher string
|
||||
contactPoint string
|
||||
|
|
@ -33,6 +35,7 @@ type configuration struct {
|
|||
|
||||
title string
|
||||
author string
|
||||
urlPrefix string
|
||||
state string
|
||||
version string
|
||||
expires string
|
||||
|
|
@ -50,6 +53,9 @@ type configuration struct {
|
|||
|
||||
loaded bool
|
||||
configPath string
|
||||
|
||||
temporal oTemporal
|
||||
spatial oSpatial
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -63,7 +69,7 @@ func NewConfiguration() configuration {
|
|||
|
||||
c.softwareName = GetSoftwareName()
|
||||
|
||||
c.title = "documentName"
|
||||
c.title = "{{documentName}}"
|
||||
c.state = "active"
|
||||
c.accessLevel = "Confidential"
|
||||
c.rights = "Restricted"
|
||||
|
|
@ -71,10 +77,14 @@ func NewConfiguration() configuration {
|
|||
c.encoding = cEncoding
|
||||
c.batchInc = 0.001
|
||||
|
||||
c.urlPrefix = "file://"
|
||||
c.hash = true
|
||||
c.hashAlgorithm = "SHA256"
|
||||
c.loaded = false
|
||||
|
||||
c.spatial = GetSpatial()
|
||||
c.temporal = GetTemporal()
|
||||
|
||||
configPath := findIni()
|
||||
if configPath != "" {
|
||||
c.LoadConfig(configPath)
|
||||
|
|
@ -130,6 +140,8 @@ func (c *configuration) SaveConfig(ConfigPath string) bool {
|
|||
cfgini, _ := ini.LooseLoad("./martilq.ini")
|
||||
|
||||
cfgini.Section("General").Key("logPath").SetValue (c.logPath)
|
||||
cfgini.Section("General").Key("tempPath").SetValue (c.tempPath)
|
||||
cfgini.Section("General").Key("dataPath").SetValue (c.dataPath)
|
||||
|
||||
cfgini.Section("MartiLQ").Key("tags").SetValue(c.tags)
|
||||
cfgini.Section("MartiLQ").Key("publisher").SetValue(c.publisher)
|
||||
|
|
@ -146,6 +158,7 @@ func (c *configuration) SaveConfig(ConfigPath string) bool {
|
|||
cfgini.Section("Resources").Key("expires").SetValue (c.expires)
|
||||
cfgini.Section("Resources").Key("encoding").SetValue (c.encoding)
|
||||
cfgini.Section("Resources").Key("version").SetValue (c.version)
|
||||
cfgini.Section("Resources").Key("urlPrefix").SetValue (c.urlPrefix)
|
||||
|
||||
cfgini.Section("Hash").Key("hashAlgorithm").SetValue (c.hashAlgorithm)
|
||||
cfgini.Section("Hash").Key("signKey_File").SetValue (c.signKey_File)
|
||||
|
|
@ -161,6 +174,13 @@ func (c *configuration) SaveConfig(ConfigPath string) bool {
|
|||
WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath))
|
||||
return false
|
||||
}
|
||||
|
||||
res := c.spatial.SaveSpatial(ConfigPath)
|
||||
res = c.temporal.SaveTemporal(ConfigPath)
|
||||
if res {
|
||||
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
@ -191,6 +211,8 @@ func (c *configuration) LoadConfig(ConfigPath string) bool {
|
|||
}
|
||||
|
||||
c.logPath = cfgini.Section("General").Key("logPath").MustString(c.logPath)
|
||||
c.tempPath = cfgini.Section("General").Key("tempPath").MustString(c.tempPath)
|
||||
c.dataPath = cfgini.Section("General").Key("dataPath").MustString(c.dataPath)
|
||||
|
||||
c.tags = cfgini.Section("MartiLQ").Key("tags").MustString(c.tags)
|
||||
c.accessLevel = cfgini.Section("MartiLQ").Key("accessLevel").MustString(c.accessLevel)
|
||||
|
|
@ -201,11 +223,12 @@ func (c *configuration) LoadConfig(ConfigPath string) bool {
|
|||
c.contactPoint = cfgini.Section("MartiLQ").Key("contactPoint").MustString(c.contactPoint)
|
||||
c.theme= cfgini.Section("MartiLQ").Key("theme").MustString(c.theme)
|
||||
|
||||
c.author = cfgini.Section("Resources").Key("title").MustString(c.title)
|
||||
c.title = cfgini.Section("Resources").Key("title").MustString(c.title)
|
||||
c.author = cfgini.Section("Resources").Key("author").MustString(c.author)
|
||||
c.state = cfgini.Section("Resources").Key("state").MustString(c.state)
|
||||
c.expires = cfgini.Section("Resources").Key("expires").MustString(c.expires)
|
||||
c.encoding = cfgini.Section("Resources").Key("encoding").MustString(c.encoding)
|
||||
c.urlPrefix = cfgini.Section("Resources").Key("urlPrefix").MustString(c.urlPrefix)
|
||||
|
||||
c.hashAlgorithm = cfgini.Section("Hash").Key("hashAlgorithm").MustString(c.hashAlgorithm)
|
||||
c.signKey_File = cfgini.Section("Hash").Key("signKey_File").MustString(c.signKey_File)
|
||||
|
|
@ -219,7 +242,9 @@ func (c *configuration) LoadConfig(ConfigPath string) bool {
|
|||
c.proxyUser = cfgini.Section("Network").Key("proxyUser").MustString(c.proxyUser)
|
||||
c.proxyCredential = cfgini.Section("Network").Key("proxyCredential").MustString(c.proxyCredential)
|
||||
|
||||
WriteLog(fmt.Sprintf("Loaded config file: %v", ConfigPath))
|
||||
c.spatial, _ = LoadSpatial(ConfigPath)
|
||||
c.temporal, _ = LoadTemporal(ConfigPath)
|
||||
|
||||
c.configPath = ConfigPath
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ package martilq
|
|||
|
||||
import (
|
||||
"time"
|
||||
"fmt"
|
||||
"gopkg.in/ini.v1"
|
||||
)
|
||||
|
||||
type oSoftware struct {
|
||||
|
|
@ -12,16 +14,18 @@ type oSoftware struct {
|
|||
}
|
||||
|
||||
type oTemporal struct {
|
||||
enabled bool
|
||||
Extension string `json:"extension"`
|
||||
BusinessDate time.Time
|
||||
RunDate time.Time
|
||||
BusinessDate time.Time `json:"businessDate"`
|
||||
RunDate time.Time `json:"runDate"`
|
||||
}
|
||||
|
||||
type oSpatial struct {
|
||||
enabled bool
|
||||
Extension string `json:"extension"`
|
||||
Country string `json:"country"`
|
||||
Region string
|
||||
Town string
|
||||
Region string `json:"region"`
|
||||
Town string `json:"town"`
|
||||
}
|
||||
|
||||
func GetSoftware() oSoftware {
|
||||
|
|
@ -48,3 +52,114 @@ func GetSpatial() oSpatial {
|
|||
|
||||
return o
|
||||
}
|
||||
|
||||
|
||||
func LoadSpatial(ConfigPath string) (oSpatial, error) {
|
||||
o := oSpatial {}
|
||||
o.Extension = "spatial"
|
||||
|
||||
cfgini, err := ini.Load(ConfigPath)
|
||||
if err != nil {
|
||||
WriteLog(fmt.Sprintf("Fail to read file: %v", ConfigPath))
|
||||
fmt.Printf("Fail to read file: %v", err)
|
||||
return o, err
|
||||
}
|
||||
|
||||
o.enabled = cfgini.Section("Custom_Spatial").Key("enabled").MustBool(o.enabled)
|
||||
o.Country = cfgini.Section("Custom_Spatial").Key("country").MustString(o.Country)
|
||||
o.Region = cfgini.Section("Custom_Spatial").Key("region").MustString(o.Region)
|
||||
o.Town = cfgini.Section("Custom_Spatial").Key("town").MustString(o.Town)
|
||||
|
||||
return o, nil
|
||||
}
|
||||
|
||||
func (s *oSpatial) SaveSpatial(ConfigPath string) bool {
|
||||
|
||||
cfgini, _ := ini.Load(ConfigPath)
|
||||
|
||||
cfgini.Section("Custom_Spatial").Key("enabled").SetValue("false")
|
||||
cfgini.Section("Custom_Spatial").Key("country").SetValue(s.Country)
|
||||
cfgini.Section("Custom_Spatial").Key("region").SetValue(s.Region)
|
||||
cfgini.Section("Custom_Spatial").Key("town").SetValue(s.Town)
|
||||
|
||||
err := cfgini.SaveTo(ConfigPath)
|
||||
if err != nil {
|
||||
WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
|
||||
}
|
||||
|
||||
|
||||
func LoadTemporal(ConfigPath string) (oTemporal, error) {
|
||||
o := oTemporal {}
|
||||
o.Extension = "temporal"
|
||||
|
||||
cfgini, err := ini.Load(ConfigPath)
|
||||
if err != nil {
|
||||
WriteLog(fmt.Sprintf("Fail to read file: %v", ConfigPath))
|
||||
fmt.Printf("Fail to read file: %v", err)
|
||||
return o, err
|
||||
}
|
||||
|
||||
o.enabled = cfgini.Section("Custom_Temporal").Key("enabled").MustBool(o.enabled)
|
||||
t := cfgini.Section("Custom_Temporal").Key("businessDate").MustString("")
|
||||
if t != "" {
|
||||
matched := false
|
||||
if t == "{{now}}" {
|
||||
matched = true
|
||||
o.BusinessDate = time.Now()
|
||||
}
|
||||
if t == "{{yesterday}}" {
|
||||
matched = true
|
||||
o.BusinessDate = time.Now().AddDate(0,0,-1)
|
||||
o.BusinessDate = time.Date(o.BusinessDate.Year(), o.BusinessDate.Month(), o.BusinessDate.Day(), 0, 0, 0, 0, time.Local)
|
||||
}
|
||||
if t == "{{today}}" {
|
||||
matched = true
|
||||
o.BusinessDate = time.Now()
|
||||
o.BusinessDate = time.Date(o.BusinessDate.Year(), o.BusinessDate.Month(), o.BusinessDate.Day(), 0, 0, 0, 0, time.Local)
|
||||
}
|
||||
if !matched {
|
||||
|
||||
}
|
||||
}
|
||||
t = cfgini.Section("Custom_Temporal").Key("runDate").MustString("")
|
||||
if t != "" {
|
||||
matched := false
|
||||
if t == "{{now}}" {
|
||||
matched = true
|
||||
o.RunDate = time.Now()
|
||||
}
|
||||
if t == "{{today}}" {
|
||||
matched = true
|
||||
o.RunDate = time.Now()
|
||||
o.RunDate = time.Date(o.RunDate.Year(), o.RunDate.Month(), o.RunDate.Day(), 0, 0, 0, 0, time.Local)
|
||||
}
|
||||
if !matched {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return o, nil
|
||||
}
|
||||
|
||||
func (s *oTemporal) SaveTemporal(ConfigPath string) bool {
|
||||
|
||||
|
||||
cfgini, _ := ini.Load(ConfigPath)
|
||||
|
||||
cfgini.Section("Custom_Temporal").Key("enabled").SetValue("false")
|
||||
cfgini.Section("Custom_Temporal").Key("businessDate").SetValue("{{today}}")
|
||||
cfgini.Section("Custom_Temporal").Key("runDate").SetValue("{{today}}")
|
||||
|
||||
err := cfgini.SaveTo(ConfigPath)
|
||||
if err != nil {
|
||||
WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,10 +55,6 @@ func NewMarti() Marti {
|
|||
|
||||
software := GetSoftware()
|
||||
m.Custom = append(m.Custom, software)
|
||||
spatial := GetSpatial()
|
||||
m.Custom = append(m.Custom, spatial)
|
||||
temporal := GetTemporal()
|
||||
m.Custom = append(m.Custom, temporal)
|
||||
|
||||
m.config = NewConfiguration()
|
||||
|
||||
|
|
@ -103,6 +99,16 @@ func (m *Marti) LoadConfig(ConfigPath string) {
|
|||
if m.config.batch != "" {
|
||||
if m.config.batch[0] == '@' {
|
||||
_, err := os.Stat(m.config.batch[1:])
|
||||
if os.IsNotExist(err) {
|
||||
// See if we can locate it in Config INI directory
|
||||
_, fileb := filepath.Split(m.config.batch[1:])
|
||||
dirc, _ := filepath.Split(ConfigPath)
|
||||
_, err := os.Stat(dirc + fileb)
|
||||
if err == nil {
|
||||
m.config.batch = "@" + dirc + fileb
|
||||
}
|
||||
}
|
||||
_, err = os.Stat(m.config.batch[1:])
|
||||
if os.IsNotExist(err) {
|
||||
WriteLog(fmt.Sprintf("Batch file '%v' does not exist" , m.config.batch))
|
||||
} else {
|
||||
|
|
@ -122,6 +128,15 @@ func (m *Marti) LoadConfig(ConfigPath string) {
|
|||
}
|
||||
}
|
||||
|
||||
if m.config.spatial.enabled == true {
|
||||
spatial := m.config.spatial
|
||||
m.Custom = append(m.Custom, spatial)
|
||||
}
|
||||
if m.config.temporal.enabled == true {
|
||||
temporal := m.config.temporal
|
||||
m.Custom = append(m.Custom, temporal)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -169,15 +184,16 @@ func (m *Marti) Save(pathFile string) bool {
|
|||
}
|
||||
|
||||
|
||||
func ProcessDirectory(ConfigPath string, SourcePath string, Recursive bool, TargetPath string) Marti {
|
||||
func ProcessFilePath(ConfigPath string, SourcePath string, Recursive bool, UrlPrefix string, DefinitionPath string) Marti {
|
||||
|
||||
m := NewMarti()
|
||||
|
||||
_, err := os.Stat(TargetPath)
|
||||
if DefinitionPath != "" {
|
||||
_, err := os.Stat(DefinitionPath)
|
||||
if err == nil {
|
||||
m, err = Load(m.config, TargetPath)
|
||||
m, err = Load(m.config, DefinitionPath)
|
||||
if err != nil {
|
||||
panic("Unable to load existing MartiLQ defintion: " + TargetPath)
|
||||
panic("Unable to load existing MartiLQ definition: " + DefinitionPath)
|
||||
}
|
||||
// Update the batch number, minor version
|
||||
m.Batch = math.Round((m.Batch + m.config.batchInc)/m.config.batchInc)*m.config.batchInc
|
||||
|
|
@ -187,6 +203,25 @@ func ProcessDirectory(ConfigPath string, SourcePath string, Recursive bool, Targ
|
|||
m.LoadConfig(ConfigPath)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if ConfigPath != "" {
|
||||
m.LoadConfig(ConfigPath)
|
||||
}
|
||||
}
|
||||
|
||||
fileStat, err := os.Stat(SourcePath)
|
||||
if err != nil {
|
||||
panic("Source path does not exist or is inaccessible: " + SourcePath)
|
||||
} else {
|
||||
|
||||
if UrlPrefix == "" {
|
||||
UrlPrefix = m.config.urlPrefix
|
||||
}
|
||||
if UrlPrefix == "" {
|
||||
UrlPrefix = "file://"
|
||||
}
|
||||
|
||||
if fileStat.IsDir() {
|
||||
|
||||
filepath.Walk(SourcePath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
|
|
@ -197,12 +232,26 @@ func ProcessDirectory(ConfigPath string, SourcePath string, Recursive bool, Targ
|
|||
|
||||
}
|
||||
} else {
|
||||
url := "file://"+info.Name()
|
||||
url := UrlPrefix+info.Name()
|
||||
m.AddResource(info.Name(), path, url)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
} else {
|
||||
url := UrlPrefix+fileStat.Name()
|
||||
m.AddResource(fileStat.Name(), SourcePath, url)
|
||||
}
|
||||
}
|
||||
|
||||
return m
|
||||
|
||||
}
|
||||
|
||||
|
||||
func ReconcileFilePath(ConfigPath string, SourcePath string, Recursive bool, DefinitionPath string, OutputPath string) Marti {
|
||||
|
||||
m := NewMarti()
|
||||
|
||||
|
||||
return m
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,8 +45,8 @@ func TestMarti_DirectoryA(t *testing.T) {
|
|||
currentDirectory, _ := os.Getwd()
|
||||
SourcePath := currentDirectory
|
||||
Recursive := false
|
||||
TargetPath := "../test/test_martilq_directoryA.json"
|
||||
ProcessDirectory("", SourcePath, Recursive, TargetPath)
|
||||
DefPath := "../test/test_martilq_directoryA.json"
|
||||
ProcessFilePath("", SourcePath, Recursive, DefPath, "")
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -55,7 +55,7 @@ func TestMarti_DirectoryB(t *testing.T) {
|
|||
currentDirectory, _ := os.Getwd()
|
||||
SourcePath := currentDirectory
|
||||
Recursive := false
|
||||
TargetPath := "../test/test_martilq_directoryB.json"
|
||||
ProcessDirectory("../config/martilq.ini", SourcePath, Recursive, TargetPath)
|
||||
DefPath := "../test/test_martilq_directoryB.json"
|
||||
ProcessFilePath("../config/martilq.ini", SourcePath, Recursive, DefPath, "")
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,15 +11,15 @@ import (
|
|||
|
||||
type Resource struct {
|
||||
Title string `json:"title"`
|
||||
Uid string `'json:"uid"`
|
||||
DocumentName string `json:"documentName`
|
||||
Uid string `json:"uid"`
|
||||
DocumentName string `json:"documentName"`
|
||||
IssueDate time.Time `json:"issueDate"`
|
||||
Modified time.Time `json:"modified"`
|
||||
Expires time.Time `json:"expires"`
|
||||
State string `json:"state"`
|
||||
Author string `json:"author"`
|
||||
Length int64 `json:"length"`
|
||||
Hash hash
|
||||
Hash hash `json:"hash"`
|
||||
|
||||
Description string `json:"description"`
|
||||
Url string `json:"url"`
|
||||
|
|
@ -28,6 +28,7 @@ type Resource struct {
|
|||
Encoding string `json:"encoding"`
|
||||
Compression string `json:"compression"`
|
||||
Encryption string `json:"encryption"`
|
||||
DescribedBy string `json:"describedBy"`
|
||||
|
||||
Attributes []Attribute `json:"attributes"`
|
||||
}
|
||||
|
|
@ -59,6 +60,11 @@ func NewMartiLQResource(config configuration, sourcePath string, urlPath string,
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
if config.dataPath != "" {
|
||||
|
||||
}
|
||||
|
||||
u := uuid.New()
|
||||
r.Uid = u.String()
|
||||
|
||||
|
|
@ -68,7 +74,7 @@ func NewMartiLQResource(config configuration, sourcePath string, urlPath string,
|
|||
r.Encoding = config.encoding
|
||||
|
||||
r.DocumentName = stats.Name()
|
||||
if config.title == "documentName" {
|
||||
if config.title == "{{documentName}}" {
|
||||
r.Title = r.DocumentName
|
||||
}
|
||||
r.IssueDate = time.Now()
|
||||
|
|
|
|||
Binary file not shown.
Loading…
Reference in New Issue