Add spanish as an example

eikek · eikek · commit 26dff18ae0d3 · 2021-01-18T17:41:40.000+01:00
Adding a new language without nlp requires now only to fill out the
pieces:

- define a list of month names to support date recognition
- add it to joex' dockerfile to be available for tesseract
- update the solr migration/field definitions
- update the elm file so it shows up on the client
diff --git a/docker/joex-base.dockerfile b/docker/joex-base.dockerfile
@@ -16,6 +16,7 @@ RUN apk add --no-cache openjdk11-jre \
     tesseract-ocr-data-deu \
     tesseract-ocr-data-fra \
     tesseract-ocr-data-ita \
+    tesseract-ocr-data-spa \
     unpaper \
     wkhtmltopdf \
     libreoffice \
diff --git a/modules/analysis/src/main/scala/docspell/analysis/date/DateFind.scala b/modules/analysis/src/main/scala/docspell/analysis/date/DateFind.scala
@@ -65,6 +65,7 @@ object DateFind {
         case Language.German  => p1.or(p0).or(p2)
         case Language.French  => p1.or(p0).or(p2)
         case Language.Italian => p1.or(p0).or(p2)
+        case Language.Spanish => p1.or(p0).or(p2)
       }
       p.read(parts) match {
         case Result.Success(sds, _) =>
diff --git a/modules/analysis/src/main/scala/docspell/analysis/date/MonthName.scala b/modules/analysis/src/main/scala/docspell/analysis/date/MonthName.scala
@@ -22,6 +22,8 @@ object MonthName {
         french
       case Language.Italian =>
         italian
+      case Language.Spanish =>
+        spanish
     }
 
   private val numbers = List(
@@ -98,4 +100,19 @@ object MonthName {
     List("nov", "novembre"),
     List("dic", "dicembre")
   )
+
+  private val spanish = List(
+    List("ene", "enero"),
+    List("feb", "febrero"),
+    List("mar", "marzo"),
+    List("abr", "abril"),
+    List("may", "mayo"),
+    List("jun"),
+    List("jul"),
+    List("ago", "agosto"),
+    List("sep", "septiembre"),
+    List("oct", "octubre"),
+    List("nov", "noviembre"),
+    List("dic", "diciembre")
+  )
 }
diff --git a/modules/common/src/main/scala/docspell/common/Language.scala b/modules/common/src/main/scala/docspell/common/Language.scala
@@ -47,7 +47,12 @@ object Language {
     val iso3 = "ita"
   }
 
-  val all: List[Language] = List(German, English, French, Italian)
+  case object Spanish extends Language {
+    val iso2 = "es"
+    val iso3 = "spa"
+  }
+
+  val all: List[Language] = List(German, English, French, Italian, Spanish)
 
   def fromString(str: String): Either[String, Language] = {
     val lang = str.toLowerCase
diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala
@@ -25,6 +25,7 @@ object Field {
   val content_en     = Field("content_en")
   val content_fr     = Field("content_fr")
   val content_it     = Field("content_it")
+  val content_es     = Field("content_es")
   val itemName       = Field("itemName")
   val itemNotes      = Field("itemNotes")
   val folderId       = Field("folder")
@@ -39,6 +40,8 @@ object Field {
         Field.content_fr
       case Language.Italian =>
         Field.content_it
+      case Language.Spanish =>
+        Field.content_es
     }
 
   implicit val jsonEncoder: Encoder[Field] =
diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrQuery.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrQuery.scala
@@ -41,6 +41,7 @@ object SolrQuery {
             Field.content_en,
             Field.content_fr,
             Field.content_it,
+            Field.content_es,
             Field.itemName,
             Field.itemNotes,
             Field.attachmentName
diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala
@@ -69,6 +69,14 @@ object SolrSetup {
             solrEngine,
             "Add content_it field",
             addContentItField.map(_ => FtsMigration.Result.reIndexAll)
+          ),
+          FtsMigration[F](
+            8,
+            solrEngine,
+            "Add content_es field",
+            addTextField(Some(Language.Spanish))(Field.content_es).map(_ =>
+              FtsMigration.Result.reIndexAll
+            )
           )
         )
 
diff --git a/modules/webapp/src/main/elm/Data/Language.elm b/modules/webapp/src/main/elm/Data/Language.elm
@@ -12,6 +12,7 @@ type Language
     | English
     | French
     | Italian
+    | Spanish
 
 
 fromString : String -> Maybe Language
@@ -27,6 +28,10 @@ fromString str =
 
     else if str == "ita" || str == "it" || str == "italian" then
         Just Italian
+
+    else if str == "spa" || str == "es" || str == "spanish" then
+        Just Spanish
+
     else
         Nothing
 
@@ -46,6 +51,9 @@ toIso3 lang =
         Italian ->
             "ita"
 
+        Spanish ->
+            "spa"
+
 
 toName : Language -> String
 toName lang =
@@ -62,7 +70,10 @@ toName lang =
         Italian ->
             "Italian"
 
+        Spanish ->
+            "Spanish"
+
 
 all : List Language
 all =
-    [ German, English, French, Italian ]
+    [ German, English, French, Italian, Spanish ]

Original file line number	Diff line number	Diff line change
`@@ -65,6 +65,7 @@ object DateFind {`
`65`	`65`	`case Language.German => p1.or(p0).or(p2)`
`66`	`66`	`case Language.French => p1.or(p0).or(p2)`
`67`	`67`	`case Language.Italian => p1.or(p0).or(p2)`
	`68`	`+ case Language.Spanish => p1.or(p0).or(p2)`
`68`	`69`	`}`
`69`	`70`	`p.read(parts) match {`
`70`	`71`	`case Result.Success(sds, _) =>`
Original file line number	Diff line number	Diff line change
`@@ -69,6 +69,14 @@ object SolrSetup {`
`69`	`69`	`solrEngine,`
`70`	`70`	`"Add content_it field",`
`71`	`71`	`addContentItField.map(_ => FtsMigration.Result.reIndexAll)`
	`72`	`+ ),`
	`73`	`+ FtsMigration[F](`
	`74`	`+ 8,`
	`75`	`+ solrEngine,`
	`76`	`+ "Add content_es field",`
	`77`	`+ addTextField(Some(Language.Spanish))(Field.content_es).map(_ =>`
	`78`	`+ FtsMigration.Result.reIndexAll`
	`79`	`+ )`
`72`	`80`	`)`
`73`	`81`	`)`
`74`	`82`