@startuml
' the image is built with https://github.com/jvantuyl/sublime_diagram_plugin
' sublimeText -plugin
' below code is PlantUML -standard (http://plantuml.com/)
' alt-m to build

' --- datasets ---
class MARC_XML << (D,yellow) >> {
    .. location ..
    COMHIS/estc-data-originals/estc-xml-raw
    .. filename ..
    originals_MARC_estc.xml
    .. dataType ..
    + XML
    .. dataSource ..
    + British Library
}

class ESTC_csv_raw << (D,yellow) >> {
    .. location ..
    COMHIS/estc-data-originals/estc-csv-raw/
    .. dataType ..
    + csv
}

class ESTC_csv_raw_filtered << (D,yellow) >> {
    .. location ..
    COMHIS/estc-data-verified/estc-csv-raw-filtered
    .. filename ..
    estc_raw_sane.csv
    .. dataType ..
    + csv
}

class ESTC_data_raw << (D,yellow) >> {
    .. location ..
    COMHIS/estc-data-verified/estc-csv-raw
    .. dataType ..
    + csv
}

class ESTC_cleaned_initial << (D,yellow) >> {
    .. location ..
    COMHIS/estc-data-unified/estc-cleaned-initial
    .. dataType ..
    + csv
    .. IdFields ..
    + estc_id
}

class publishers_unified << (D,yellow) >> {
    .. location ..
    COMHIS/estc-data-unified/estc-publishers
    .. dataType ..
    + csv
    .. IdFields ..
    + estc_id
    + actor_id
}

class physicaldimensions_unified << (D,yellow) >> {
    .. location ..
    COMHIS/estc-data-unified/estc-physicaldimension
    .. dataType ..
    + csv
    .. languages ..
    + R 3.5.1
    
}

class language_unified << (D,yellow) >> {
    .. location ..
    COMHIS/estc-data-unified/estc-language
    .. dataType ..
    + csv
    .. languages ..
    + R 3.5.1
    
}

class physicaldimensions_summaries << (A,green) >> {
    .. location ..
    COMHIS/estc-physicaldimension
    .. dataType ..
    + markdown
    + csv    
    .. languages ..
    + R 3.5.1
    
}

class actor_summaries << (A,green) >> {
    .. location ..
    COMHIS/author_analysis
    .. dataType ..
    + markdown
    + csv    
    .. languages ..
    + R 3.5.1
    
}

class physical_extent_unified << (D,yellow) >> {
    .. location ..
    COMHIS/estc-data-unified/estc-physical-extent
    .. dataType ..
    + csv
    + rds
    .. languages ..
    + R 3.5.1
    
}

class ESTC_actor_data_unified << (D,yellow) >> {
    .. location ..
    COMHIS/estc-data-unified/estc-viaf
    .. dataType ..
    + csv
}

class BBTI_data << (D,yellow) >> {
    .. location ..
    http://bbti.bodleian.ox.ac.uk
    .. dataType ..
    + website
}

class VIAF_XML << (D,yellow) >> {
    .. location ..
    http://viaf.org/viaf/data/
    .. dataType ..
    + XML
    .. dataSource ..
    + viaf.org
}

class ECCO_data_raw << (D,yellow) >> {
    .. location ..
    COMHIS/ecco-data-private/originals
    .. dataType ..
    + json
}

'  --- data processing scripts ---
class XML_parser << (P,violet) >> {
    .. repository ..
    COMHIS/MARCdata
    .. languages ..
    + C++
}

class estc_xml2csv << (P,violet) >> {
    .. repository ..
    COMHIS/estc-xml2csv
    .. languages ..
    + python
    .. authors ..
    + Hege
}

class ESTC_precleaner << (P,violet) >> {
    .. repository ..
    COMHIS/estc-raw-csv-prepicker
    .. filename ..
    prefilter_main.py
    .. languages ..
    + python
    .. authors ..
    + Ville
}

class ESTC_master << (P,violet) >> {
    .. repository ..
    COMHIS/estc
    .. languages ..
    + R
}

class unify_publishers << (P,violet) >> {
    .. repository ..
    COMHIS/unify_publishers
    .. languages ..
    + R 3.4.3
    + Python 3.6.3

}

class unify_physicaldimension << (P,violet) >> {
    .. repository ..
    COMHIS/estc-physicaldimension
    .. languages ..
    + R 3.5.1
}

class unify_physicalextent << (P,violet) >> {
    .. repository ..
    COMHIS/estc-physical-extent
    .. languages ..
    + R 3.5.1
}


class unify_language << (P,violet) >> {
    .. repository ..
    COMHIS/estc-language
    .. languages ..
    + R 3.5.1
}


' --- analysis scripts ---
class ESTC_overview_analysis << (A,lightgreen) >> {
    .. repository ..
    COMHIS/estc-overview-analysis
    .. languages ..
    + R
}


' Mark's boxes
class unify_actors << (P,violet) >> {
    .. repository ..
    + COMHIS/estc-viaf
    .. languages ..
    + R
}

class author_analysis << (P,violet) >> {
    .. repository ..
    + COMHIS/author_analysis
    .. languages ..
    + R
}

class estc_author_analysis_data << (P,violet) >> {
    .. repository ..
    + COMHIS/estc_author_analysis_data
    .. languages ..
    + R
}

class ESTC_actors_enriched << (D,yellow) >> {
    .. location ..
    COMHIS/estc-author-analysis-data
    .. dataType ..
    + csv
}

class ESTC_actors_verified << (D,yellow) >> {
    .. location ..
    COMHIS/estc-author-analysis-data
    .. dataType ..
    + csv
}

' Authors (and other actors)
VIAF_XML --> unify_actors
MARC_XML --> unify_actors
unify_actors --> ESTC_actor_data_unified
ESTC_actor_data_unified --> actor_summaries
ESTC_actor_data_unified --> author_analysis
author_analysis --> ESTC_actors_enriched
ESTC_actors_enriched --> estc_author_analysis_data
estc_author_analysis_data --> ESTC_actors_verified

' raw data wrangling
MARC_XML --> XML_parser
XML_parser --> ESTC_data_raw

MARC_XML --> estc_xml2csv
estc_xml2csv --> ESTC_csv_raw

ESTC_csv_raw --> ESTC_precleaner
ESTC_precleaner --> ESTC_csv_raw_filtered

ESTC_data_raw --> ESTC_master
ESTC_master --> ESTC_cleaned_initial
ESTC_cleaned_initial --> unify_publishers

' Publishers
BBTI_data --> unify_publishers
unify_publishers --> publishers_unified

' Physical dimension
ESTC_data_raw --> unify_physicaldimension
unify_physicaldimension --> physicaldimensions_summaries
unify_physicaldimension --> physicaldimensions_unified

' Language
ESTC_data_raw --> unify_language
unify_language --> language_summaries
unify_language --> language_unified


' Physical extent
ESTC_data_raw --> unify_physicalextent
ECCO_data_raw --> unify_physicalextent
unify_physicalextent --> physical_extent_unified

' Analysis
ESTC_cleaned_initial --> ESTC_overview_analysis
physicaldimensions_unified --> ESTC_overview_analysis
physical_extent_unified --> ESTC_overview_analysis
language_unified --> ESTC_overview_analysis
ESTC_actor_data_unified --> ESTC_overview_analysis


hide empty members
' hide circle
@enduml
