# Standard Pipelines
In diesem Bereich werden die Standard Pipelines des digitalen Lesesaals dokumentiert. Diese gelten als Referenz beim Implementieren neuer Pipelines.
# 01_records
[
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR",
"TITEL",
"PARENT_ID_NR",
"ZWEIG_POSITION",
"SIGNATUR",
"STUFE",
"ENTSTEHUNGSZEITRAUM",
"ENTSTEHUNGSZEITRAUM_BEGIN",
"ENTSTEHUNGSZEITRAUM_ENDE",
"DARIN",
"ARCHIVALIENART",
"ANZAHL",
"LAUFMETER",
"ENTHAELT",
"AKTENBILDNER_PROVENIENZ_TEXT",
"INHALT",
"BESTANDSGESCHICHTE",
"RECHTSSTATUS",
"COPYRIGHT_REPORDUKTIONSBESTIMM",
"VERFERTIGERIN",
"FORM_UND_INHALT",
"ORDNUNG_UND_KLASSIFIKATION",
"ARCHIVGESCHICHTE",
"AUTOR",
"BEARBEITER_IN",
"GESCHICHTE",
"HIERARCHIE_PFAD",
"ERFASST_AM",
"MUTIERT_AM",
"SCHUTZFRISTENDE",
"BASISDATUM_NM",
"BASISDATUM_ID",
"SCHUTZFRISTDAUER",
"BEWILLIGUNG_NM",
"BEWILLIGUNG_ID",
"PHYSISCHE_BENUETZBARKEIT_NM",
"PHYSISCHE_BENUETZBARKEIT_ID",
"ZUGAENGLICHKEIT_NM",
"ZUGAENGLICHKEIT_ID",
"SCHUTZFRISTKATEGORIE_NM",
"SCHUTZFRISTKATEGORIE_ID",
"SCHUTZFRISTNOTIZ",
"ANZEIGE_INTERNET"
],
"page_size": 100,
"filter_sql": "\"HIERARCHIE_PFAD\" NOT LIKE '00000796390000546102%'",
"table_name": "public.mvk_dls_vrzng_enht_1",
"order_columns": [
"ID_NR"
]
}
},
{
"section": "Map",
"options": {
"operations": [
{
"target": "_today",
"function": "now"
},
{
"args": [
"$_today",
"%Y-%m-%d"
],
"target": "_today",
"function": "strftime"
}
]
}
},
{
"section": "Map",
"options": {
"operations": [
{
"args": [
"$ZUGAENGLICHKEIT_ID",
"==",
10007
],
"target": "files_are_public",
"function": "condition"
}
]
}
},
{
"section": "Map",
"options": {
"operations": [
{
"args": [
"$PHYSISCHE_BENUETZBARKEIT_ID",
"in",
[
1
]
],
"target": "is_orderable",
"function": "condition"
}
]
}
},
{
"section": "Map",
"options": {
"operations": [
{
"args": [
"and",
[
"$SCHUTZFRISTENDE",
"is not",
null
],
[
"$SCHUTZFRISTENDE",
">",
"$_today"
]
],
"target": "is_approval_required",
"function": "condition"
}
]
}
},
{
"section": "Map",
"options": {
"operations": [
{
"args": [
"open",
[
"and",
["$PHYSISCHE_BENUETZBARKEIT_NM", "in", ["uneingeschränkt", "Digitalisat vorhanden", "Freihandbereich Lesesaal"]],
["$ZUGAENGLICHKEIT_NM", "in", ["Oeffentlich", "Freigegeben für Online-Zugänglichkeit"]]
]
],
"target": "usability_accessibility_type",
"function": "setif"
},
{
"args": [
"impossible",
[
"and",
["$usability_accessibility_type", "is", null],
["or",
["$PHYSISCHE_BENUETZBARKEIT_NM", "==", "nicht möglich"],
["$ZUGAENGLICHKEIT_NM", "==", "Nicht öffentlich (neu)"]
]
]
],
"target": "usability_accessibility_type",
"function": "setif"
},
{
"args": [
"limited",
["$usability_accessibility_type", "is", null]
],
"target": "usability_accessibility_type",
"function": "setif"
}
]
}
},
{
"section": "Map",
"options": {
"operations": [
{
"args": [
"$SCHUTZFRISTENDE",
[
"and",
[
"$SCHUTZFRISTENDE",
"is not",
null
],
[
"$SCHUTZFRISTENDE",
"<",
"0001-01-01"
]
]
],
"target": "protection_period_end",
"function": "nullif"
}
]
}
},
{
"section": "Map",
"options": {
"operations": [
{
"args": [
"$ENTSTEHUNGSZEITRAUM_BEGIN",
[
"and",
[
"$ENTSTEHUNGSZEITRAUM_BEGIN",
"is not",
null
],
[
"$ENTSTEHUNGSZEITRAUM_BEGIN",
"<",
"0001-01-01"
]
]
],
"target": "created_start",
"function": "nullif"
},
{
"args": [
"$ENTSTEHUNGSZEITRAUM_ENDE",
[
"and",
[
"$ENTSTEHUNGSZEITRAUM_ENDE",
"is not",
null
],
[
"$ENTSTEHUNGSZEITRAUM_ENDE",
"<",
"0001-01-01"
]
]
],
"target": "created_end",
"function": "nullif"
}
]
}
},
{
"section": "Map",
"options": {
"operations": [
{
"function": "date_to_int",
"args": ["$created_start"],
"target": "created_start"
},
{
"function": "date_to_int",
"args": ["$created_end"],
"target": "created_end"
}
]
}
},
{
"section": "Map",
"options": {
"operations": [
{
"args": [
"$ARCHIVALIENART",
"\r\n\r\n"
],
"target": "archival_type",
"function": "split"
}
]
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "ID_NR",
"target": "identifier"
},
{
"source": "PARENT_ID_NR",
"target": "parent_identifier"
},
{
"source": "ERFASST_AM",
"target": "ais_created"
},
{
"source": "MUTIERT_AM",
"target": "ais_modified"
},
{
"source": "protection_period_end",
"target": "protection_period_end"
},
{
"source": "ANZEIGE_INTERNET",
"target": "is_public"
},
{
"source": "files_are_public",
"target": "files_are_public"
},
{
"source": "is_approval_required",
"target": "is_approval_required"
},
{
"source": "is_orderable",
"target": "is_orderable"
},
{
"source": "usability_accessibility_type",
"target": "usability_accessibility_type"
},
{
"source": "TITEL",
"target": "title"
},
{
"source": "SIGNATUR",
"target": "signature"
},
{
"source": "STUFE",
"target": "description_level"
},
{
"source": "ENTSTEHUNGSZEITRAUM",
"target": "created_display"
},
{
"source": "created_start",
"target": "created_start"
},
{
"source": "created_end",
"target": "created_end"
},
{
"source": "archival_type",
"target": "archival_type"
},
{
"source": "VERFERTIGERIN",
"target": "author"
},
{
"source": "BEARBEITER_IN",
"target": "editor"
},
{
"source": "RECHTSSTATUS",
"target": "legal_state"
},
{
"source": "AKTENBILDNER_PROVENIENZ_TEXT",
"target": "provenance"
},
{
"source": "INHALT",
"target": "description"
},
{
"source": "ENTHAELT",
"target": "contains"
},
{
"source": "DARIN",
"target": "contains_also"
},
{
"source": "FORM_UND_INHALT",
"target": "content_form"
},
{
"source": "BASISDATUM_NM",
"target": "protection_base_date"
},
{
"source": "SCHUTZFRISTDAUER",
"target": "protection_duration"
},
{
"source": "SCHUTZFRISTKATEGORIE_NM",
"target": "protection_period_category"
},
{
"source": "BEWILLIGUNG_NM",
"target": "approval"
},
{
"source": "ZUGAENGLICHKEIT_NM",
"target": "accessibility"
},
{
"source": "COPYRIGHT_REPORDUKTIONSBESTIMM",
"target": "copyright"
},
{
"source": "PHYSISCHE_BENUETZBARKEIT_NM",
"target": "usability"
},
{
"custom": true,
"source": "ANZAHL",
"label_de": "Anzahl",
"display_group": 1,
"slug": "quantity"
},
{
"custom": true,
"source": "ARCHIVGESCHICHTE",
"label_de": "Archivgeschichte",
"display_group": 3,
"slug": "archive_history"
},
{
"custom": true,
"source": "AUTOR",
"label_de": "Autor",
"display_group": 3,
"slug": "custom_author"
},
{
"custom": true,
"source": "BESTANDSGESCHICHTE",
"label_de": "Bestandsgeschichte",
"display_group": 3,
"slug": "bestand_history"
},
{
"custom": true,
"source": "GESCHICHTE",
"label_de": "Geschichte",
"display_group": 3,
"slug": "history"
},
{
"custom": true,
"source": "LAUFMETER",
"label_de": "Laufmeter",
"display_group": 1,
"slug": "linear_meter"
},
{
"custom": true,
"source": "SCHUTZFRISTNOTIZ",
"label_de": "Schutzfristnotiz",
"display_group": 4,
"slug": "protection_period_notice"
}
]
}
},
{
"section": "CreateRecords",
"options": {
"prune": false,
"batch_size": 100
}
},
{
"section": "MissingSourceItems",
"options": {
"model": "record",
"identifier_field": "identifier"
}
},
{
"section": "DeleteItems",
"options": {
"model": "record",
"identifier_field": "identifier"
}
}
]
# 02_cleanup_records
[
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR"
],
"page_size": 500,
"table_name": "public.mvk_dls_vrzng_enht_1",
"order_columns": [
"ID_NR"
]
}
},
{
"section": "MissingSourceItems",
"options": {
"model": "record",
"identifier_field": "ID_NR"
}
},
{
"section": "DeleteItems",
"options": {
"model": "record",
"identifier_field": "identifier"
}
}
]
# 03_tree
[
{
"section": "Tree",
"options": {
"root_value": "-1"
}
},
{
"section": "Solr"
}
]
# 04_records_records
[
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR_1",
"ID_NR_2",
"ROLLE",
"GEGENROLLE"
],
"page_size": 100,
"table_name": "public.mvk_dls_vrzng_enht_bzhng_1",
"order_columns": [
"ID_NR_1",
"ID_NR_2"
]
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "record",
"lookup": "identifier",
"source": "ID_NR_1",
"target": "from_record_id",
"missing": "drop"
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "record",
"lookup": "identifier",
"source": "ID_NR_2",
"target": "to_record_id",
"missing": "drop"
}
},
{
"section": "Map",
"options": {
"operations": [
{
"function": "join",
"args": ["$from_record_id", "$to_record_id"],
"target": "identifier"
}
]
}
},
{
"section": "Deduplicate",
"options": {
"field": "identifier"
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "identifier",
"target": "identifier"
},
{
"source": "from_record_id",
"target": "from_record_id"
},
{
"source": "to_record_id",
"target": "to_record_id"
},
{
"source": "ROLLE",
"target": "role"
},
{
"source": "GEGENROLLE",
"target": "reverse_role"
}
]
}
},
{
"section": "CreateRecordRelations",
"options": {
"prune": true,
"batch_size": 100
}
}
]
# 05_files
[
{
"section": "QuerysetSource",
"options": {
"model": "record",
"filter": {
"is_public": true,
"files_are_public": true
}
}
},
{
"section": "Logger",
"options": {
"prefix": "Record",
"fields": [
"identifier"
],
"oneline": true
}
},
{
"section": "ScopeArchivSource",
"options": {
"page_size": 100,
"table_name": "public.mvk_dls_ve_datei_1",
"columns": [
"ID_NR_DATEI",
"ID_NR_VRZNG_ENHT",
"DATEINAME",
"VERZEICHNIS",
"URL"
],
"order_columns": [
"ID_NR_VRZNG_ENHT",
"ID_NR_DATEI"
],
"filter_by_previous_in": {
"previous_field": "identifier",
"column": "ID_NR_VRZNG_ENHT"
}
}
},
{
"section": "FileDeduplicator"
},
{
"section": "SkipProcessedFiles",
"options": {
"identifier_field": "ID_NR_DATEI",
"pipeline_version": "1.0.0"
}
},
{
"section": "Logger",
"options": {
"prefix": "File",
"fields": [
"ID_NR_DATEI",
"ID_NR_VRZNG_ENHT",
"URL"
],
"oneline": true
}
},
{
"section": "FileDownloader",
"options": {
"identifier_field": "ID_NR_DATEI",
"source_path_field": "URL",
"source_path_regexp_replace": [
"file:///[A-Z]:(.*)",
"\\1"
],
"config_name": "opendls-s3"
}
},
{
"section": "FilePublisher",
"options": {}
},
{
"section": "Filter",
"options": {
"keep": [
"$public_path"
]
}
},
{
"section": "Logger",
"options": {
"prefix": "Published",
"fields": [
"ID_NR_DATEI",
"public_path"
],
"oneline": true
}
},
{
"section": "ResolveForeignKey",
"options": {
"source": "ID_NR_VRZNG_ENHT",
"model": "record",
"lookup": "identifier",
"target": "record_id"
}
},
{
"section": "Map",
"options": {
"operations": [
{
"function": "setvalue",
"args": ["fileshare"],
"target": "source"
}
]
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "identifier",
"target": "identifier"
},
{
"source": "ID_NR_DATEI",
"target": "collection_identifier"
},
{
"source": "source",
"target": "source"
},
{
"source": "record_id",
"target": "record_id"
},
{
"source": "public_path",
"target": "s3_path"
},
{
"source": "searchable_text",
"target": "searchable_text"
},
{
"source": "media_type",
"target": "media_type"
},
{
"source": "metadata",
"target": "metadata"
},
{
"source": "DATEINAME",
"target": "original_filename"
},
{
"source": "ID_NR_VRZNG_ENHT",
"target": "_record_identifier"
},
{
"source": "pipeline_version",
"target": "pipeline_version"
}
]
}
},
{
"section": "CreateIIIFSources"
},
{
"section": "Publish",
"options": {
"model": "file"
}
},
{
"section": "Solr",
"options": {
"prune": false,
"batch_size": 50,
"identifier_field": "_record_identifier"
}
}
]
# 06_fedora_files
[
{
"section": "QuerysetSource",
"options": {
"model": "record",
"filter": {
"is_public": true,
"files_are_public": true
}
}
},
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"PID",
"ID_NR_VRZNG_ENHT"
],
"page_size": 100,
"table_name": "public.mvk_dls_ve_vrkng_fdra_1",
"order_columns": [
"PID",
"ID_NR_VRZNG_ENHT"
],
"filter_by_previous_in": {
"column": "ID_NR_VRZNG_ENHT",
"previous_field": "identifier"
}
}
},
{
"section": "FileDownloader",
"options": {
"source_type": "fedora",
"identifier_field": "PID",
"source_path_field": "PID",
"config_name": "opendls-fedora"
}
},
{
"section": "Logger",
"options": {
"prefix": "AIP"
}
},
{
"section": "FilePublisher",
"options": {}
},
{
"section": "Filter",
"options": {
"keep": [
"$public_path"
]
}
},
{
"section": "Logger",
"options": {
"prefix": "Published"
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "record",
"lookup": "identifier",
"source": "ID_NR_VRZNG_ENHT",
"target": "record_id"
}
},
{
"section": "Map",
"options": {
"operations": [
{
"function": "split",
"args": [
"$identifier",
"/"
],
"target": "original_filename"
},
{
"function": "setvalue",
"args": ["$original_filename.-1"],
"target": "original_filename"
}
]
}
},
{"section": "Logger"},
{
"section": "Map",
"options": {
"operations": [
{
"function": "setvalue",
"args": ["fedora"],
"target": "source"
}
]
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "public_path",
"target": "identifier"
},
{
"source": "PID",
"target": "collection_identifier"
},
{
"source": "source",
"target": "source"
},
{
"source": "record_id",
"target": "record_id"
},
{
"source": "public_path",
"target": "s3_path"
},
{
"source": "searchable_text",
"target": "searchable_text"
},
{
"source": "media_type",
"target": "media_type"
},
{
"source": "metadata",
"target": "metadata"
},
{
"source": "ID_NR_VRZNG_ENHT",
"target": "_record_identifier"
},
{
"source": "original_filename",
"target": "original_filename"
}
]
}
},
{
"section": "CreateIIIFSources",
"options": {
"prune": false,
"batch_size": 10
}
},
{
"section": "Publish",
"options": {
"model": "file",
"prune": false
}
},
{
"section": "Solr",
"options": {
"prune": false,
"batch_size": 50,
"identifier_field": "_record_identifier"
}
}
]
# 07_cleanup_files
[
{
"section": "QuerysetSource",
"options": {
"model": "record",
"filter": {
"is_public": true,
"files_are_public": true
}
}
},
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR_DATEI"
],
"page_size": 500,
"table_name": "public.mvk_dls_ve_datei_1",
"order_columns": [
"ID_NR_DATEI"
],
"filter_by_previous_in": {
"column": "ID_NR_VRZNG_ENHT",
"previous_field": "identifier"
}
}
},
{
"section": "MissingSourceItems",
"options": {
"model": "file",
"fields": [
"identifier",
"record_id"
],
"filter": {
"source": "fileshare"
},
"identifier_field": "ID_NR_DATEI"
}
},
{
"section": "DeleteItems",
"options": {
"model": "file",
"identifier_field": "identifier"
}
},
{
"section": "Solr",
"options": {
"prune": false,
"batch_size": 50,
"lookup_field": "pk",
"identifier_field": "record_id"
}
}
]
# 08_cleanup_fedora_files
[
{
"section": "QuerysetSource",
"options": {
"model": "record",
"filter": {
"is_public": true,
"files_are_public": true
},
"batch_size": 100
}
},
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"PID"
],
"page_size": 100,
"table_name": "public.mvk_dls_ve_vrkng_fdra_1",
"order_columns": [
"PID"
],
"filter_by_previous_in": {
"column": "ID_NR_VRZNG_ENHT",
"previous_field": "identifier"
}
}
},
{
"section": "MissingSourceItems",
"options": {
"model": "file",
"fields": [
"collection_identifier",
"record_id"
],
"filter": {
"source": "fedora"
},
"identifier_field": "PID",
"lookup_field": "collection_identifier"
}
},
{
"section": "Logger",
"options": {
"limit": 100,
"fields": null,
"prefix": "Logger",
"oneline": false
}
},
{
"section": "DeleteItems",
"options": {
"model": "file",
"identifier_field": "collection_identifier",
"lookup_field": "collection_identifier"
}
},
{
"section": "Solr",
"options": {
"prune": false,
"batch_size": 50,
"lookup_field": "pk",
"identifier_field": "record_id"
}
}
]
# 10_descriptors
[
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR",
"THESAURUS",
"BEZEICHNUNG",
"BESCHREIBUNG",
"ERFASST_AM",
"MUTIERT_AM"
],
"page_size": 100,
"table_name": "public.mvk_dls_dskrp_1",
"order_columns": [
"ID_NR"
]
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "BESCHREIBUNG",
"target": "description"
},
{
"source": "BEZEICHNUNG",
"target": "title"
},
{
"source": "ID_NR",
"target": "identifier"
},
{
"source": "THESAURUS",
"target": "thesaurus"
}
]
}
},
{
"section": "CreateDescriptors",
"options": {
"prune": true,
"batch_size": 100
}
}
]
# 11_descriptors_descriptors
[
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR_1",
"ID_NR_2",
"ROLLE",
"GEGENROLLE"
],
"page_size": 100,
"table_name": "public.mvk_dls_dskrp_bzhng_1",
"order_columns": [
"ID_NR_1",
"ID_NR_2"
]
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "descriptor",
"lookup": "identifier",
"source": "ID_NR_1",
"target": "from_descriptor_id",
"missing": "drop"
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "descriptor",
"lookup": "identifier",
"source": "ID_NR_2",
"target": "to_descriptor_id",
"missing": "drop"
}
},
{
"section": "Map",
"options": {
"operations": [
{
"function": "join",
"args": ["$from_descriptor_id", "$to_descriptor_id"],
"target": "identifier"
}
]
}
},
{
"section": "Deduplicate",
"options": {
"field": "identifier"
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "identifier",
"target": "identifier"
},
{
"source": "from_descriptor_id",
"target": "from_descriptor_id"
},
{
"source": "to_descriptor_id",
"target": "to_descriptor_id"
},
{
"source": "ROLLE",
"target": "role"
},
{
"source": "GEGENROLLE",
"target": "reverse_role"
}
]
}
},
{
"section": "CreateDescriptorRelations",
"options": {
"prune": true,
"batch_size": 100
}
}
]
# 12_descriptors_records
[
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR_DSKRP",
"ID_NR_VRZNG_ENHT"
],
"page_size": 100,
"table_name": "public.mvk_dls_ve_dskrp_bzhng_1",
"order_columns": [
"ID_NR_DSKRP",
"ID_NR_VRZNG_ENHT"
]
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "record",
"lookup": "identifier",
"source": "ID_NR_VRZNG_ENHT",
"target": "record_id",
"missing": "drop"
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "descriptor",
"lookup": "identifier",
"source": "ID_NR_DSKRP",
"target": "descriptor_id",
"missing": "drop"
}
},
{
"section": "Map",
"options": {
"operations": [
{
"function": "join",
"args": ["$record_id", "$descriptor_id"],
"target": "identifier"
}
]
}
},
{
"section": "Deduplicate",
"options": {
"field": "identifier"
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "identifier",
"target": "identifier"
},
{
"source": "record_id",
"target": "record_id"
},
{
"source": "descriptor_id",
"target": "descriptor_id"
}
]
}
},
{
"section": "CreateDescriptorRecordRelations",
"options": {
"prune": true,
"batch_size": 100
}
}
]
# 13_publish_descriptors
[
{
"section": "QuerysetSource",
"options": {
"model": "descriptor"
}
},
{
"section": "Publish",
"options": {
"model": "descriptor",
"prune": true,
"batch_size": 200
}
}
]
# 14_publish_descriptors_descriptors
[
{
"section": "QuerysetSource",
"options": {
"model": "descriptorrelation"
}
},
{
"section": "Publish",
"options": {
"model": "descriptorrelation",
"prune": true,
"batch_size": 200
}
}
]
# 15_publish_descriptors_records
[
{
"section": "QuerysetSource",
"options": {
"model": "descriptorrecordrelation"
}
},
{
"section": "Publish",
"options": {
"model": "descriptorrecordrelation",
"prune": true,
"batch_size": 200
}
}
]
# 20_deliveries
[
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR",
"ID_NAME",
"ABLIEFERNDE_STELLE",
"AKTENBILDENDE_STELLE",
"ZEITRAUM_PERIODE"
],
"page_size": 100,
"table_name": "public.mvk_dls_ablfr_1",
"order_columns": [
"ID_NR"
]
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "ID_NR",
"target": "identifier"
},
{
"source": "ID_NAME",
"target": "title"
},
{
"source": "ABLIEFERNDE_STELLE",
"target": "delivery_source"
},
{
"source": "AKTENBILDENDE_STELLE",
"target": "creator"
},
{
"source": "ZEITRAUM_PERIODE",
"target": "period"
}
]
}
},
{
"section": "CreateDeliveries",
"options": {
"prune": true,
"batch_size": 100
}
}
]
# 21_deliveries_records
[
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR_ABLFR",
"ID_NR_VRZNG_ENHT"
],
"page_size": 100,
"table_name": "public.mvk_dls_ve_ablfr_bzhng_1",
"order_columns": [
"ID_NR_VRZNG_ENHT",
"ID_NR_ABLFR"
]
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "record",
"lookup": "identifier",
"source": "ID_NR_VRZNG_ENHT",
"target": "record_id",
"missing": "drop"
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "delivery",
"lookup": "identifier",
"source": "ID_NR_ABLFR",
"target": "delivery_id",
"missing": "drop"
}
},
{
"section": "Map",
"options": {
"operations": [
{
"function": "join",
"args": ["$record_id", "$delivery_id"],
"target": "identifier"
}
]
}
},
{
"section": "Deduplicate",
"options": {
"field": "identifier"
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "identifier",
"target": "identifier"
},
{
"source": "record_id",
"target": "record_id"
},
{
"source": "delivery_id",
"target": "delivery_id"
}
]
}
},
{
"section": "CreateDeliveryRecordRelations",
"options": {
"prune": true,
"batch_size": 100
}
}
]
# 22_publish_deliveries
[
{
"section": "QuerysetSource",
"options": {
"model": "delivery"
}
},
{
"section": "Publish",
"options": {
"model": "delivery",
"prune": true,
"batch_size": 200
}
}
]
# 23_publish_deliveries_records
[
{
"section": "QuerysetSource",
"options": {
"model": "deliveryrecordrelation"
}
},
{
"section": "Publish",
"options": {
"model": "deliveryrecordrelation",
"prune": true,
"batch_size": 200
}
}
]
# 30_containers
[
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR",
"TYP",
"DEF_STANDORT"
],
"page_size": 100,
"table_name": "public.mvk_dls_bhltns_1",
"order_columns": [
"ID_NR"
]
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "ID_NR",
"target": "identifier"
},
{
"source": "TYP",
"target": "type"
},
{
"source": "DEF_STANDORT",
"target": "location"
}
]
}
},
{
"section": "CreateContainers",
"options": {
"prune": true,
"batch_size": 100
}
}
]
# 31_containers_records
[
{
"section": "ScopeArchivSource",
"options": {
"columns": [
"ID_NR_VRZNG_ENHT",
"ID_NR_BHLTNS"
],
"page_size": 100,
"table_name": "public.mvk_dls_ve_bhltns_bzhng_1",
"order_columns": [
"ID_NR_VRZNG_ENHT",
"ID_NR_BHLTNS"
]
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "record",
"lookup": "identifier",
"source": "ID_NR_VRZNG_ENHT",
"target": "record_id",
"missing": "drop"
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "container",
"lookup": "identifier",
"source": "ID_NR_BHLTNS",
"target": "container_id",
"missing": "drop"
}
},
{
"section": "Map",
"options": {
"operations": [
{
"function": "join",
"args": ["$record_id", "$container_id"],
"target": "identifier"
}
]
}
},
{
"section": "Deduplicate",
"options": {
"field": "identifier"
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "identifier",
"target": "identifier"
},
{
"source": "record_id",
"target": "record_id"
},
{
"source": "container_id",
"target": "container_id"
}
]
}
},
{
"section": "CreateContainerRecordRelations",
"options": {
"prune": true,
"batch_size": 100
}
}
]
# 32_publish_containers
[
{
"section": "QuerysetSource",
"options": {
"model": "container"
}
},
{
"section": "Publish",
"options": {
"model": "container",
"prune": true,
"batch_size": 200
}
}
]
# 33_publish_containers_records
[
{
"section": "QuerysetSource",
"options": {
"model": "containerrecordrelation"
}
},
{
"section": "Publish",
"options": {
"model": "containerrecordrelation",
"prune": true,
"batch_size": 200
}
}
]
# 40_download_from_url
[
{
"section": "RawSource",
"options": {
"data": [
{
"dpi": 94,
"url": "https://d37dhr5745n0y2.cloudfront.net/b/3c/ea/b3cea5825822f4d0da778dda9b642b0d633a73b9_765745782.jpg",
"label": "original",
"width": 2500,
"height": 1667,
"record_identifier": "124886"
}
]
}
},
{
"section": "Map",
"options": {
"operations": [
{
"args": [
"$url"
],
"target": "identifier",
"function": "generate_identifier"
}
]
}
},
{
"section": "SkipProcessedFiles",
"options": {
"fedora": false,
"batch_size": 50,
"identifier_field": "identifier",
"pipeline_version": "1.0.0"
}
},
{
"section": "FileDownloader",
"options": {
"source_type": "http",
"identifier_field": "identifier",
"source_path_field": "url"
}
},
{
"section": "FilePublisher",
"options": {
"package_processing": false
}
},
{
"section": "Logger",
"options": {
"limit": 100,
"fields": null,
"prefix": "Published",
"oneline": true
}
},
{
"section": "ResolveForeignKey",
"options": {
"model": "record",
"lookup": "identifier",
"source": "record_identifier",
"target": "record_id",
"missing": "raise"
}
},
{
"section": "Map",
"options": {
"operations": [
{
"args": [
"fileshare"
],
"target": "source",
"function": "setvalue"
}
]
}
},
{
"section": "AttributeMapper",
"options": {
"mapping": [
{
"source": "identifier",
"target": "identifier",
"protected": false,
"omit_empty": false
},
{
"source": "record_identifier",
"target": "collection_identifier",
"protected": false,
"omit_empty": false
},
{
"source": "source",
"target": "source",
"protected": false,
"omit_empty": false
},
{
"source": "record_id",
"target": "record_id",
"protected": false,
"omit_empty": false
},
{
"source": "public_path",
"target": "s3_path",
"protected": false,
"omit_empty": false
},
{
"source": "searchable_text",
"target": "searchable_text",
"protected": false,
"omit_empty": false
},
{
"source": "media_type",
"target": "media_type",
"protected": false,
"omit_empty": false
},
{
"source": "metadata",
"target": "metadata",
"protected": false,
"omit_empty": false
},
{
"source": "original_filename",
"target": "original_filename",
"protected": false,
"omit_empty": false
},
{
"source": "record_identifier",
"target": "_record_identifier",
"protected": false,
"omit_empty": false
},
{
"source": "pipeline_version",
"target": "pipeline_version",
"protected": false,
"omit_empty": false
}
]
}
},
{
"section": "CreateIIIFSources",
"options": {
"prune": false,
"batch_size": 100
}
},
{
"section": "Publish",
"options": {
"model": "file",
"prune": false,
"batch_size": 200
}
},
{
"section": "Solr",
"options": {
"prune": false,
"batch_size": 50,
"lookup_field": "identifier",
"identifier_field": "_record_identifier"
}
}
]
# 90_solr
[
{
"section": "QuerysetSource",
"options": {
"model": "record",
"filter": {
"is_public": true
}
}
},
{
"section": "Solr",
"options": {
"prune": false,
"batch_size": 500,
"identifier_field": "identifier"
}
}
]
← ETL Pipeline Sections →