From 01b812b69c58ca0cea57a9f0f776bc853c59115c Mon Sep 17 00:00:00 2001 From: Daniel Cartwright Date: Thu, 29 Oct 2020 10:56:50 -0700 Subject: [PATCH 001/136] Update dependencies/CI Summary: This PR accomplishes several things: - removes dist-newstyle (local build artifacts should not be checked in) - extends the .gitignore to include many common build artifacts/editor artifacts - allow more modern dependencies (upper bounds of many were out of date by one or two years' worth of releases) - upgrade stack lts (9.2 -> 14.2) to GHC 8.6.5 - regenerate .travis.yml using the now-standard haskell-ci (many haskell core libraries use this), instead of the outdated script that was maintained by hvr; as a precursor to this, the tested-with versions were updated Reviewed By: patapizza Differential Revision: D24623967 fbshipit-source-id: 838fe571df0b8d44106349659ce8ce8ab82f0bc6 --- .gitignore | 50 ++++++++- .travis.yml | 210 +++++++++++++++++++++++-------------- dist-newstyle/cache/config | Bin 69929 -> 0 bytes duckling.cabal | 23 ++-- stack.yaml | 2 +- 5 files changed, 189 insertions(+), 96 deletions(-) delete mode 100644 dist-newstyle/cache/config diff --git a/.gitignore b/.gitignore index 510f52aaa..615382e8e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,49 @@ +# object files +*.a +*.o +*.so + +# haskell stuff +.ghc.environment.* +dist +dist-* +cabal-dev +*.chi +*.chs.h +*.dyn_o +*.dyn_hi +*.prof +*.hp +*.eventlog +.cabal-sandbox/ +cabal.sandbox.config +cabal.config +cabal.project.local +.HTF/ .stack-work/ -log/ -.idea/ +stack.yaml.lock + +# vim stuff +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-v][a-z] +[._]sw[a-p] +*~ + +# nix stuff +result +result-* +*.hi + +# make/latexmk stuff +.makefile +*.aux +*.fdb_latexmk +*.fls +*.log +*.out + +# misc +tags +log +.idea diff --git a/.travis.yml b/.travis.yml index 339bc7c1b..dadb9cdfd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,92 +1,140 @@ -# This file has been generated -- see https://github.com/hvr/multi-ghc-travis +# This Travis job script has been generated by a script via +# +# haskell-ci 'travis' 'duckling.cabal' +# +# To regenerate the script (for example after adjusting tested-with) run +# +# haskell-ci regenerate +# +# For more information, see https://github.com/haskell-CI/haskell-ci +# +# version: 0.10.3 +# +version: ~> 1.0 language: c -sudo: false - +os: linux +dist: xenial +git: + # whether to recursively clone submodules + submodules: false cache: directories: - - $HOME/.cabsnap - $HOME/.cabal/packages - + - $HOME/.cabal/store + - $HOME/.hlint before_cache: - - rm -fv $HOME/.cabal/packages/hackage.haskell.org/build-reports.log - - rm -fv $HOME/.cabal/packages/hackage.haskell.org/00-index.tar - -matrix: + - rm -fv $CABALHOME/packages/hackage.haskell.org/build-reports.log + # remove files that are regenerated by 'cabal update' + - rm -fv $CABALHOME/packages/hackage.haskell.org/00-index.* + - rm -fv $CABALHOME/packages/hackage.haskell.org/*.json + - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.cache + - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar + - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar.idx + - rm -rfv $CABALHOME/packages/head.hackage +jobs: include: - # crashes in travis because 2 versions of unix get installed and - # when text-show tries to use TH the wrong one gets linked - # - env: CABALVER=1.24 GHCVER=7.10.3 HAPPYVER=1.19.5 BUILD=cabal - # compiler: ": #GHC 7.10.3" - # addons: {apt: {packages: [cabal-install-1.24,ghc-7.10.3,happy-1.19.5], sources: [hvr-ghc]}} - - env: CABALVER=1.24 GHCVER=8.0.2 - compiler: ": #GHC 8.0.2" - addons: {apt: {packages: [cabal-install-1.24,ghc-8.0.2,happy-1.19.5], sources: [hvr-ghc]}} - - env: CABALVER=2.0 GHCVER=8.2.2 - compiler: ": #GHC 8.2.2" - addons: {apt: {packages: [cabal-install-2.0,ghc-8.2.2,happy-1.19.5], sources: [hvr-ghc]}} - - env: CABALVER=2.0 GHCVER=8.6.3 - compiler: ": #GHC 8.6.3" - addons: {apt: {packages: [cabal-install-2.0,ghc-8.6.3,happy-1.19.5], sources: [hvr-ghc]}} - + - compiler: ghc-8.6.5 + addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.6.5","cabal-install-3.2"]}} + os: linux + - compiler: ghc-8.4.4 + addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.4.4","cabal-install-3.2"]}} + os: linux before_install: - - unset CC - - export HAPPYVER=1.19.5 - - export PATH=$HOME/.cabal/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH - - export PATH=/opt/happy/$HAPPYVER/bin:$PATH - + - HC=$(echo "/opt/$CC/bin/ghc" | sed 's/-/\//') + - WITHCOMPILER="-w $HC" + - HADDOCK=$(echo "/opt/$CC/bin/haddock" | sed 's/-/\//') + - HCPKG="$HC-pkg" + - unset CC + - CABAL=/opt/ghc/bin/cabal + - CABALHOME=$HOME/.cabal + - export PATH="$CABALHOME/bin:$PATH" + - TOP=$(pwd) + - "HCNUMVER=$(${HC} --numeric-version|perl -ne '/^(\\d+)\\.(\\d+)\\.(\\d+)(\\.(\\d+))?$/; print(10000 * $1 + 100 * $2 + ($3 == 0 ? $5 != 1 : $3))')" + - echo $HCNUMVER + - CABAL="$CABAL -vnormal+nowrap" + - set -o pipefail + - TEST=--enable-tests + - BENCH=--enable-benchmarks + - HEADHACKAGE=false + - rm -f $CABALHOME/config + - | + echo "verbose: normal +nowrap +markoutput" >> $CABALHOME/config + echo "remote-build-reporting: anonymous" >> $CABALHOME/config + echo "write-ghc-environment-files: always" >> $CABALHOME/config + echo "remote-repo-cache: $CABALHOME/packages" >> $CABALHOME/config + echo "logs-dir: $CABALHOME/logs" >> $CABALHOME/config + echo "world-file: $CABALHOME/world" >> $CABALHOME/config + echo "extra-prog-path: $CABALHOME/bin" >> $CABALHOME/config + echo "symlink-bindir: $CABALHOME/bin" >> $CABALHOME/config + echo "installdir: $CABALHOME/bin" >> $CABALHOME/config + echo "build-summary: $CABALHOME/logs/build.log" >> $CABALHOME/config + echo "store-dir: $CABALHOME/store" >> $CABALHOME/config + echo "install-dirs user" >> $CABALHOME/config + echo " prefix: $CABALHOME" >> $CABALHOME/config + echo "repository hackage.haskell.org" >> $CABALHOME/config + echo " url: http://hackage.haskell.org/" >> $CABALHOME/config install: - - cabal --version - - echo "$(ghc --version) [$(ghc --print-project-git-commit-id 2> /dev/null || echo '?')]" - - if [ -f $HOME/.cabal/packages/hackage.haskell.org/00-index.tar.gz ]; - then - zcat $HOME/.cabal/packages/hackage.haskell.org/00-index.tar.gz > - $HOME/.cabal/packages/hackage.haskell.org/00-index.tar; - fi - - travis_retry cabal update -v - - sed -i 's/^jobs:/-- jobs:/' ${HOME}/.cabal/config - - cabal install --only-dependencies --enable-tests --enable-benchmarks --dry -v > installplan.txt - - sed -i -e '1,/^Resolving /d' installplan.txt; cat installplan.txt - -# check whether current requested install-plan matches cached package-db snapshot - - if diff -u $HOME/.cabsnap/installplan.txt installplan.txt; - then - echo "cabal build-cache HIT"; - rm -rfv .ghc; - cp -a $HOME/.cabsnap/ghc $HOME/.ghc; - cp -a $HOME/.cabsnap/lib $HOME/.cabsnap/share $HOME/.cabsnap/bin $HOME/.cabal/; - else - echo "cabal build-cache MISS"; - rm -rf $HOME/.cabsnap; - mkdir -p $HOME/.ghc $HOME/.cabal/lib $HOME/.cabal/share $HOME/.cabal/bin; - cabal install -j1 --only-dependencies --enable-tests --enable-benchmarks; - fi - -# snapshot package-db on cache miss - - if [ ! -d $HOME/.cabsnap ]; - then - echo "snapshotting package-db to build-cache"; - mkdir $HOME/.cabsnap; - cp -a $HOME/.ghc $HOME/.cabsnap/ghc; - cp -a $HOME/.cabal/lib $HOME/.cabal/share $HOME/.cabal/bin installplan.txt $HOME/.cabsnap/; - fi - -# Here starts the actual work to be performed for the package under test; -# any command which exits with a non-zero exit code causes the build to fail. + - ${CABAL} --version + - echo "$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]" + - | + echo "program-default-options" >> $CABALHOME/config + echo " ghc-options: $GHCJOBS +RTS -M6G -RTS" >> $CABALHOME/config + - cat $CABALHOME/config + - rm -fv cabal.project cabal.project.local cabal.project.freeze + - travis_retry ${CABAL} v2-update -v + # Generate cabal.project + - rm -rf cabal.project cabal.project.local cabal.project.freeze + - touch cabal.project + - | + echo "packages: ." >> cabal.project + - echo 'package duckling' >> cabal.project + - "echo ' ghc-options: -Werror=missing-methods' >> cabal.project" + - | + - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(duckling)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done" + - cat cabal.project || true + - cat cabal.project.local || true + - if [ -f "./configure.ac" ]; then (cd "." && autoreconf -i); fi + - ${CABAL} v2-freeze $WITHCOMPILER ${TEST} ${BENCH} + - "cat cabal.project.freeze | sed -E 's/^(constraints: *| *)//' | sed 's/any.//'" + - rm cabal.project.freeze + - travis_wait 40 ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} --dep -j2 all + - travis_wait 40 ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks --dep -j2 all script: - - if [ -f configure.ac ]; then autoreconf -i; fi - - cabal configure --enable-tests -v2 # -v2 provides useful information for debugging - - cabal build -j1 # this builds all libraries and executables (including tests/benchmarks) - - travis_wait cabal test -j1 --show-details=always - - cabal check - - cabal sdist # tests that a source-distribution can be generated - -# Check that the resulting source distribution can be built & installed. -# If there are no other `.tar.gz` files in `dist`, this can be even simpler: -# `cabal install --force-reinstalls dist/*-*.tar.gz` - -# Building it again doubles the time and it's currently timing out on Travis. -# It's also not that useful at the moment. -#- SRC_TGZ=$(cabal info . | awk '{print $2;exit}').tar.gz && -# (cd dist && cabal install --force-reinstalls "$SRC_TGZ") + - DISTDIR=$(mktemp -d /tmp/dist-test.XXXX) + # Packaging... + - ${CABAL} v2-sdist all + # Unpacking... + - mv dist-newstyle/sdist/*.tar.gz ${DISTDIR}/ + - cd ${DISTDIR} || false + - find . -maxdepth 1 -type f -name '*.tar.gz' -exec tar -xvf '{}' \; + - find . -maxdepth 1 -type f -name '*.tar.gz' -exec rm '{}' \; + - PKGDIR_duckling="$(find . -maxdepth 1 -type d -regex '.*/duckling-[0-9.]*')" + # Generate cabal.project + - rm -rf cabal.project cabal.project.local cabal.project.freeze + - touch cabal.project + - | + echo "packages: ${PKGDIR_duckling}" >> cabal.project + - echo 'package duckling' >> cabal.project + - "echo ' ghc-options: -Werror=missing-methods' >> cabal.project" + - | + - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(duckling)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done" + - cat cabal.project || true + - cat cabal.project.local || true + # Building... + # this builds all libraries and executables (without tests/benchmarks) + - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all + # Building with tests and benchmarks... + # build & run tests, build benchmarks + - ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} all + # Testing... + - ${CABAL} v2-test $WITHCOMPILER ${TEST} ${BENCH} all + # cabal check... + - (cd ${PKGDIR_duckling} && ${CABAL} -vnormal check) + # haddock... + - ${CABAL} v2-haddock $WITHCOMPILER --with-haddock $HADDOCK ${TEST} ${BENCH} all + # Building without installed constraints for packages in global-db... + - rm -f cabal.project.local + - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all +# REGENDATA ("0.10.3",["travis","duckling.cabal"]) # EOF diff --git a/dist-newstyle/cache/config b/dist-newstyle/cache/config deleted file mode 100644 index 2ab20b6617fa439ec736d2271fa660f75ebe9f90..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69929 zcmeHQ-)|$yR(726Br~(KyRdu3y$C@h?gLz|<2XN@WOb1?lVm5^Ofo}~*}c0D3~jre zm~m`hcYAl{O8f=9Ao0ZIg@1u3UJydO@Pver0A0j8Nc;u7QEm4*?W(Sh`+U{=OH4)@ zJ8^yIJ5^nE>Z?=L?NVv-voL)rRm$b~-Mg!`_PF)3*X>`ew!Eg-D=+-({$Kvf_ul!t ze;WVo|LhOD|FuX(AJv`*{wSz5TZ7Rs@M{Ny*7(Nnhe2(B(Dpwaj9%8xo;+i7wTZda z;b`#3ek)w<4O(8W!p8QUjCDr7|8qaF_7fLtNn`wmVP@JQwbr2D>0YUZIe!07|NR$# zUHP-W|Bt`8w2R&VY?fI)xQ5_5WepDx1d!U{PG_k6D)pR^k3bEQH;N+{<(?i z4X$qC*?%?o;6Lvgm`dmeVQ?F2_}*WAIW$nwuQONX7I;)S%k_{V-qwc{xpQmxiVHdQ zBWB||K7WvIRW5;jkU(k+Iq+LTJNa+4S12MsqE;{9)bqcAMVDdTVRF(XO{zyLGRr3^m&ujZMF`7@)lfCuU9ZP>7Wwk=C5S(njt}~H>7Uh~ zr@zpj==U5QK7p|D=^OfB{*Y0feELHc)Nl0Ge5ImXWTxcgEq`Dlli%3}{pl7AmrcDP zaQwB@_v$)a`JLA9 zuB@-sH><;uA1LYaQ-CKc=CjOQm&G<4!*vq~+U`@x|1#XMtL zn`Bi*E3{0DmbfyxK$VNo3$Eu>zj8kA@%70xJ6q`P{@~_@n<5vaymvm&@mYrQBNFQQ ztUo#vS>L}96+hUM6+aXeKYSu9F01OHtg45XB4<82mKB$Eg2%FUKa#ckk<9laneRu> zMJ;oDAS*7rs>iac`mwCtPlUx!MO8hK_4kvfBI{3N#ZP5p^;G8j>1QI}&mPH&%ZA69 ztg6q1#brJ6Ty~kxWrOeH6H!B6d@3q_DH|S_M>6ZOu=M;yWc?FajQo6_?@Q-i|7C_W z%$n&9%>Bbx;mB)+i5ANVyc2QHslCDVV87=DLATRY+2a^VYoL_q)oW%!R`$+c{Pd^p zW*$8*#>`nEX7(=%F>|n2jG4ni%p5)`#?0A^eH#1@{pRrO#bXNfZf2g7Y;ZGEL@z!p zrr#W17ILLNIxfacF(dqOF@xc#n89#VOxqk4(>6!X3%OFq2gR5vrWYR<(~CbYrr(?t zWahMxCO#=_kX1an$U+njz@NZXt}D#lDPVe9P4i$}ELahUFu8D0sw znJHq{d{#`qc~(rkIWJ}~oEH<)FFq-x`(1omh?&b`g6QQ@G2Rq2`941>#G6lwQ52sa zz1XJ&i^I@VW{zl8>}KYavJ`G+E-Am@W`^TU)?UT^gRVLS6{v+YEpD?`(&>`S zZSF>gg=g+=hXpft%fp5tT_R|=xtkvrp1C_9HVhCG=kA4Ac;;@1STMXRlBMS!?%fQ= z{qSk$bkO%-$2$axJY+WbEQX6CY(#UU5?9vTO|iEK3zPC5ZSd(X zIh!lQxksPJwJOFrHdlyqydhDHb8Jp?PS(HpvKHeK@0Ap6POBl>(~fDrD#kfBSB!Hs z(HG<#nJdJ(k4cFY;~bkS#JRI#hdi4r#JOj*D_=~@u{q5-aL8*m&_Q)cLaUexvbjQ> z<2~16oMUr^IQKbiPZ#4Ho70>Vb`4rV>Mnss931(GB&CZZY{bD4&PBL5!bTh%;RJ_^ zBW%RQks>Y)ALw*&gi|CgEy6}z9H9_cgbkOf{I+ad3n)G%k*?5eG*&?c?GI8*y>OzOa_fzMPP9aD)?7E-k`F z930_{kBcL0#K93xBe^)jMl?sV5_;A&%+itH8zu*0MOd6w+c?bXj>`w?SpZ(Yug zvrhTiEE}&mo|~R`20?npWAq31WJ7NE8IRFtGajUuEhJLcp0_kHdRG46wP!p= z&w3xEXFNvF;u@r9JVwu3iXc7XG5QmGt}1uWXFNzxsuip~OJ|IpwJJ!@c#NL6_dt5a zWArQsL3+kx^eh5Fdd7qFGX0nex-#jQg0tWQYtMMB_Ls7}M9O`}WArTUz}hojqt6m} zruR8@p$UVyG&Pvz;6qY_&->kwwof%=c0eAFyqP=9qZB(A|AU*A3q4b2u=vfAW^o+;oSq6gijK}C@$>P+o<6Hbe?->tz zPtySim(Ha|$LBMD7`;>j8|-{S4{Up3A~|*K8IRSTH8AQvX>g35H84oec#NL2ydXW} zL3+yVqV$Bv==rc8NY8kXp7wG;dY166c+7p4@F+cnJ&c|uJV?)YjGiStNY8kTp3gLb z^o+;oc?F@2v_l_`fhC}3>F~GG$DgECpcDf%3y-z1c%7^-e!!P;Sf3BB#n_X z93!WT29gsTBc~_Og5(5;$T?|@kuw}4C)E#<6C5L_d^Jc;aEP3<)fhR$F>;!gKyrdZ zYR*}9h@6w$7&*f+a(a#cNKSBwoF7Plkuw}4x8I4GH<`r`M!?8P z^@BAhI7H6-3K%)VA#y%_kC8JRBG1?o$$QNiu90Wuhm8VV2Nnc43y#L{p{>tU1A< znsc%XBWE~9P7^XnPH>2v^Gz5z!!dG7`GDjEhsZhQgOM{FBc~7rk`o*v=adgd&Tx#} zZB`RzjEo?|DIcun42NpYSr?3);TSnd5s;kV5IHA4Fmi@Nj>?q-dGPF7J>*jzv1|GxI#((@VRx$ut-4vc5Ss5GBlY$(H|M?DfNLh(4J$8?U*Yb+J-#KG3T-mmH7L^Ku-dXFNvFPX++#8IRGk zR0ru957LuZN9hTV(erIZAU)$TdcHLuq-Q)vFAR~Xfy;P|p4B5*d&YzGGWnURy-a_m z;5?y&wP(Cm`>Z*}q|d1cOqe#mWN~R9O~6jD3c{UW6Sxzkg&dq96R;C9MP%y|YyqzZ3zGs&z%9Y@1b2c> zz)tY(x408*0(OF>3ho4((45FpAeL^GQ}9@jC}S?=bLUchXB{Yc>{!j+EUlwK`SeA; z%VWm^v~=zQp(U)GmQP#&w2TF4X>SIhC9FnUiMx%u>Adx_*X>`$ziZcr)u;3;X69LO zafQDDMsEvYh1iOVEBt*gPOh*O7gvPe^OA9a*ouoQ{5332t-@BET%o@2<_cMHafQEP z#mN=6;^GQ_p^B3$Y{kVD{`wRrSJ;Y+EBr+#POh*O7gzXeO`Kd|D=x0^mzX%Y!d5g_ za_I{`>8Cp&!^&wn-2l)sR!++a27s2aa$4R70B9Kt(2@{CXbB6@%5t~bTq!#Zr$ISe z0o0bU^4juQE`XM?04*g05L&{@Y55=#K+9NwmW~i1w1m}YbLm;$!YMzZ;h;N!T5)lO zozjft3fO!YO0R_s3((RrD1?@oH^Xu0x8sYG2cx^12az*#I_sD?^|>6SK|faq$>N$+o_V8ye#x3U?nR)- zBd^ZtJZDl-{2|r*3sq_wFZyo_eNiN_FV<$(pva(3-0`!NMmv z8`O+Os99@3wPrL#O{du~YCW5o(rC5H+I_YF17VHKP$~7CjI(qakY2`xrH$5o(r75H+JU z>MY5jPkT8}HZAr* z@k|Z9m6R&oZw>mP%FLpJ7Zwm`hv;tc+CRDk{(T=l;mQ@?TY{ z(HBa87&rPt=>-=mmEvN<)@Wia`mi#gTqym~OtCwm@@ndn@_BM;%Fz|%pLoS_D2oD$G^WiXz)R;||7CI_xNhWu@~?^eI;hS&7R->=dR>i4RjcKa>0 zTh{YKzdBTtp+D-={t8zu;|+DGW$IE?ZZa>^FYl_U<)z>2RfADW^|o*lfXk)#qR&+* zEth_aN=AQ(Vm|$DjmGpLy?!g4c%NE8RnUtXb7olG_SxHq)8gNOp+$WzYna?Q@Io)X zDCVvuU8}U({pz$N`O9hm9wY31#v9U!w-1A>Aj%u4V9jm$}y#G*)WC6$lQZTDJ7A<@JCZ?N=(J7 zjIMc9P8*?#M2kL(ekPOO^lhp1HcPKTkLB}3c&dRPxsm*sDvdtL6$9Q}0VR^D#lDd% z@cq^}2nRRSw$j@DpgV|k#Uv2MUd{Xb2m2HJ0m0+A_$`+k)a3m7O=tibMPFHbqt#X_ zzUd)NoAhb@N-d+W4Ct@C>ZCu+2`;&U!FSBvUt{So@_#l~8H8#uJ8HmMFgi6g!eo{kC8(j9m^p)ucZBTwAv&)-inIsG;O44hxVE13C zOkmLcf)=IOpS({2`}8`tR5eTFaVZq7qev3TG*frKnJn7hKv4)IxYJ zYH@GajoEXt+&%1HscR#F+~*Y$RY7B-tq8Ny5BF{c<9_(GbE@_|(vP!ru#ucknr&B^ zAIWKQ+(vSu8n=<0I>K!v$A@qm$?0eZ8+^hUw~>4!frAacXAHNIoK|+Q@zgfVWc@Tb zID`9hdaE0K>jcyQJrX1C( zN$+(h!U$);r4i16%cNq;d1-(&*ppk_pq7`$SA$&|Uro3$z8Y|8fHdIJ@NB@7o7Z3vW7BrX6+cu}JypSXoTfv3Av!CG$0GuGTfa%U3w}nP}P6 zmqOFQFvxS+!ouWE9ukme9Hh{Pl0}V)md*Gnv{S3eQduc(QNqPUCl3}t+FUlOOm3M_ zG10Q5SqgpZ(pzN9xD>Z&vNF-KBwY$EilHgAQ(u#5JCj=`?NVs^-cBg4oj8qenW{8B ze=Hn`qh6!9eG`PWm?|O0}(l7HV4qE!3AL zS{6|Zv`}A~XxaQ@qGfZwiI&A911(gD23n{P4YW`pnrN9qG|{5%So53ZWjQz#EmVkx z=l0cZ*6xnUE!21hw@~AmXqm<{(K4N7poL1wMDt^EbIo1`TBwvvv=g^b>loa!q@mf@ zgj&bo7HS;>Ez~*&TBue`v{P&m>J5Wis5cC>P;cnyTzbRuUBg1nVY6WGr&$?5i-lai zo8KuZbUX!@na%W@8QWvN3MLa}5;BcaaIlip_#-xL7TSgtpCsY|Pm#$RxAPf@}xZYC$wX*(}IJsLg^* zh}ta3Vt~zpEQVPvh(xK)f-IKUEXcx%&4MgJVzVF%Di#a5;>zrkx`hEngUS*OI#iTp z(4evugANtt7j&p7=4nt_u+yNjD5pVXxhcXc{HeOJ*q=R zi%|_KTZ!sWQHo20%7UW?l`SDPsB8_VL1ha#4Jr%5vrsMiW-XhvhYio%vy-KJ_2&=@ zv5EQ_KBSQgO?uuN2$U{O>vz%nUifMsgZ0L#{s z23QuNOt5G%Xn^gyFKhhk;9Mai`%#-S@gv^avpj0^Yk2q&J%w?(s!q`LtKA^>W;sgq zMz>eSzr!~PJ3YXQo&b6K;|7e!e9S(fOM4o?txxBugkBKFPvU$_Jy>Y^=r4-xQ7b%{ zl&D^h``yW7As0&bqkE`T5&h28xW)MKmETF{sS>5N+H-ZQX;5pb=QF7Xe%1~KE%gNb aei+pD2W|h;!RTe}?8&nl-LzX=1.10 tested-with: - GHC==8.0.2, - GHC==8.2.2, - GHC==8.6.3 + GHC==8.4.4, + GHC==8.6.5 extra-source-files: README.md @@ -829,17 +828,17 @@ library build-depends: base >= 4.8.2 && < 5.0 , array >= 0.5.1.1 && < 0.6 , attoparsec >= 0.13.1.0 && < 0.14 - , aeson >= 0.11.3.0 && < 1.5 + , aeson >= 0.11.3.0 && < 1.6 , bytestring >= 0.10.6.0 && < 0.11 , containers >= 0.5.6.2 && < 0.7 , deepseq >= 1.4.1.1 && < 1.5 , dependent-sum >= 0.3.2.2 && < 0.5 - , extra >= 1.4.10 && < 1.7 - , hashable >= 1.2.4.0 && < 1.3 - , regex-base >= 0.93.2 && < 0.94 - , regex-pcre >= 0.94.4 && < 0.95 + , extra >= 1.4.10 && < 1.8 + , hashable >= 1.2.4.0 && < 1.4 + , regex-base >= 0.93.2 && < 0.95 + , regex-pcre >= 0.94.4 && < 0.96 , text >= 1.2.2.1 && < 1.3 - , text-show >= 2.1.2 && < 3.8 + , text-show >= 2.1.2 && < 3.9 , time >= 1.5.0.1 && < 2 , timezone-series >= 0.1.5.1 && < 0.2 , unordered-containers >= 0.2.7.2 && < 0.3 @@ -1160,7 +1159,7 @@ executable duckling-example-exe , text , text-show , time - , timezone-olson >= 0.1.7 && < 0.2 + , timezone-olson >= 0.1.7 && < 0.3 , timezone-series , unordered-containers default-language: Haskell2010 @@ -1195,7 +1194,7 @@ executable duckling-request-sample , filepath >= 1.4.0.0 && < 1.5 , text , time - , timezone-olson >= 0.1.7 && < 0.2 + , timezone-olson >= 0.1.7 && < 0.3 , timezone-series , unordered-containers default-language: Haskell2010 @@ -1214,7 +1213,7 @@ executable duckling-expensive , filepath >= 1.4.0.0 && < 1.5 , text , time - , timezone-olson >= 0.1.7 && < 0.2 + , timezone-olson >= 0.1.7 && < 0.3 , timezone-series , unordered-containers default-language: Haskell2010 diff --git a/stack.yaml b/stack.yaml index e68eb9168..30fee675b 100644 --- a/stack.yaml +++ b/stack.yaml @@ -1,4 +1,4 @@ -resolver: lts-9.10 +resolver: lts-14.20 packages: - '.' From bfc75849d2d1cd239fa242902329d0a9d5173d3b Mon Sep 17 00:00:00 2001 From: Victor Pothin Date: Mon, 2 Nov 2020 12:05:36 -0800 Subject: [PATCH 002/136] Adds new rules of accentuation of the Portuguese (#531) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Keeps accents consistent, "quinquagésimo" there is no more "Ü". Pull Request resolved: https://github.com/facebook/duckling/pull/531 Reviewed By: patapizza Differential Revision: D23770703 Pulled By: chessai fbshipit-source-id: f8a34c02028faf9f51eca6a016b5bad988a83f04 --- Duckling/Ordinal/PT/Corpus.hs | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/Duckling/Ordinal/PT/Corpus.hs b/Duckling/Ordinal/PT/Corpus.hs index cd80ff987..00aa568ec 100644 --- a/Duckling/Ordinal/PT/Corpus.hs +++ b/Duckling/Ordinal/PT/Corpus.hs @@ -34,34 +34,50 @@ allExamples = concat ] , examples (OrdinalData 7) [ "setimo" + , "sétimo" , "sétimas" ] , examples (OrdinalData 10) - [ "décimos" - , "decimos" + [ "decimos" + , "décimos" + , "decima" , "décima" , "decimas" + , "décimas" ] , examples (OrdinalData 11) - [ "décimos primeiros" + [ "decimos primeiros" + , "décimos primeiros" , "decimo primeiro" + , "décimo primeiro" + , "decimas primeiras" , "décimas primeiras" , "decima primeira" + , "décima primeira" ] , examples (OrdinalData 12) - [ "décimos segundos" + [ "decimos segundos" + , "décimos segundos" , "decimo segundo" + , "décimo segundo" + , "decimas segundas" , "décimas segundas" , "decima segunda" + , "décima segunda" ] , examples (OrdinalData 17) - [ "décimos setimos" - , "decimo sétimo" - , "decimas sétimas" + [ "decimos setimos" + , "décimos setimos" + , "decimo setimo" + , "décimo sétimo" + , "decimas setimas" + , "décimas sétimas" , "decima setima" + , "décima setima" ] , examples (OrdinalData 58) - [ "quinquagesimas oitavas" + [ "quinquagésimas oitavas" , "qüinquagesimo oitavo" + , "quinquagésimo oitavo" ] ] From 888b1cba35cb44592f28bc1d505d1a1e681bfab0 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 2 Nov 2020 13:21:40 -0800 Subject: [PATCH 003/136] Dockerfile: debugs the build and uses Debian Buster everywhere (#539) Summary: The Dockerfile build part did not copy the Duckling implementation into the container, making the build fail. I also harmonized the target Debian to Buster, that is the one currently hidden behind `haskell:8`. Pull Request resolved: https://github.com/facebook/duckling/pull/539 Reviewed By: patapizza Differential Revision: D24688839 Pulled By: chessai fbshipit-source-id: 0ffcc4d28a599b7edad668730117828d26e116ad --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3576547b8..7027a5857 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM haskell:8 AS builder +FROM haskell:8-buster AS builder RUN apt-get update -qq && \ apt-get install -qq -y libpcre3 libpcre3-dev build-essential --fix-missing --no-install-recommends && \ @@ -9,7 +9,7 @@ RUN mkdir /log WORKDIR /duckling -ADD stack.yaml . +ADD . . ENV LANG=C.UTF-8 @@ -23,7 +23,7 @@ ADD . . # '-j1' flag to force the build to run sequentially. RUN stack install -FROM debian:stretch +FROM debian:buster ENV LANG C.UTF-8 From eb043d70186d30a258061455323a1218c5cd063b Mon Sep 17 00:00:00 2001 From: Daniel Cartwright Date: Mon, 9 Nov 2020 11:09:48 -0800 Subject: [PATCH 004/136] Quantity rules for Spanish (ES) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Spanish (ES) will now have all the same quantity rules as English (EN) (which I think is the most-supported language), plus more. This includes the following: * bowls - (bol(es)?|tazón(es)?|cuencos?|platos? (soperos?)|(hondos?)) (EN does not currently have this) * cups - (tazas?) * dishes - (platos?|fuentes?) (EN does not currently have this) * grams - (((m(ili)?)|(k(ilo)?))?g(ramo)?s?) * ounces - ((onzas?)|oz) * pints - (pintas?) (EN does not currently have this) * pounds - ((lb|libra)s?) * quarts - (cuartos? de galón) (EN does not currently have this) * tablespoons - (cucharadas? (grande)?) (EN does not currently have this) * teaspoons - (cucharaditas?) (EN does not currently have this) Reviewed By: patapizza Differential Revision: D24628214 fbshipit-source-id: 2e8d500661f30fa0928cb7d3f21470afc01e2285 --- Duckling/Dimensions/ES.hs | 1 + Duckling/Quantity/ES/Corpus.hs | 86 ++++++++++ Duckling/Quantity/ES/Rules.hs | 258 ++++++++++++++++++++++++++++ Duckling/Rules/ES.hs | 3 +- duckling.cabal | 3 + tests/Duckling/Quantity/ES/Tests.hs | 22 +++ tests/Duckling/Quantity/Tests.hs | 3 +- 7 files changed, 374 insertions(+), 2 deletions(-) create mode 100644 Duckling/Quantity/ES/Corpus.hs create mode 100644 Duckling/Quantity/ES/Rules.hs create mode 100644 tests/Duckling/Quantity/ES/Tests.hs diff --git a/Duckling/Dimensions/ES.hs b/Duckling/Dimensions/ES.hs index 7f6a639b6..e0662d7fe 100644 --- a/Duckling/Dimensions/ES.hs +++ b/Duckling/Dimensions/ES.hs @@ -17,6 +17,7 @@ allDimensions = , Seal Duration , Seal Numeral , Seal Ordinal + , Seal Quantity , Seal Temperature , Seal Time , Seal Volume diff --git a/Duckling/Quantity/ES/Corpus.hs b/Duckling/Quantity/ES/Corpus.hs new file mode 100644 index 000000000..337c5b7f5 --- /dev/null +++ b/Duckling/Quantity/ES/Corpus.hs @@ -0,0 +1,86 @@ +-- Copyright (c) 2016-present, Facebook, Inc. +-- All rights reserved. +-- +-- This source code is licensed under the BSD-style license found in the +-- LICENSE file in the root directory of this source tree. + + +{-# LANGUAGE OverloadedStrings #-} + +module Duckling.Quantity.ES.Corpus + ( corpus + ) where + +import Prelude +import Data.String + +import Duckling.Locale +import Duckling.Quantity.Types +import Duckling.Resolve +import Duckling.Testing.Types + +context :: Context +context = testContext { locale = makeLocale ES Nothing } + +corpus :: Corpus +corpus = (context, testOptions, allExamples) + +allExamples :: [Example] +allExamples = concat + [ examples (simple Pound 2 (Just "carne")) + [ "dos libras de carne" + ] + , examples (simple Gram 2 Nothing) + [ "dos gramos" + , "0,002 kg" + , "2/1000 kilogramos" + , "2000 miligramos" + ] + , examples (simple Gram 1000 Nothing) + [ "un kilogramo" + , "un kg" + ] + , examples (simple Pound 1 Nothing) + [ "una libra" + , "1 lb" + , "una lb" + ] + , examples (simple Ounce 2 Nothing) + [ "2 onzas" + , "2oz" + ] + , examples (simple Cup 3 (Just "azucar")) + [ "tres tazas de azucar" + , "3 tazas de AzUcAr" + ] + , examples (simple Cup 0.75 Nothing) + [ "3/4 taza" + , "0,75 taza" + , ",75 tazas" + ] + , examples (simple Gram 500 (Just "fresas")) + [ "500 gramos de fresas" + , "500g de fresas" + , "0,5 kilogramos de fresas" + , "0,5 kg de fresas" + , "500000mg de fresas" + ] + , examples (under Pound 6 (Just "carne")) + [ "menos que seis libras de carne" + , "no más que 6 lbs de carne" + , "por debajo de 6,0 libras de carne" + , "a lo sumo seis libras de carne" + ] + , examples (above Cup 2 Nothing) + [ "excesivo 2 tazas" + , "como mínimo dos tazas" + , "mayor de 2 tazas" + , "más de 2 tazas" + ] + , examples (above Ounce 4 (Just "chocolate")) + [ "excesivo 4 oz de chocolate" + , "al menos 4,0 oz de chocolate" + , "mayor de cuatro onzas de chocolate" + , "más de cuatro onzas de chocolate" + ] + ] diff --git a/Duckling/Quantity/ES/Rules.hs b/Duckling/Quantity/ES/Rules.hs new file mode 100644 index 000000000..0e2d66446 --- /dev/null +++ b/Duckling/Quantity/ES/Rules.hs @@ -0,0 +1,258 @@ +-- Copyright (c) 2016-present, Facebook, Inc. +-- All rights reserved. +-- +-- This source code is licensed under the BSD-style license found in the +-- LICENSE file in the root directory of this source tree. + + +{-# LANGUAGE GADTs #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE LambdaCase #-} + +module Duckling.Quantity.ES.Rules + ( rules + ) where + +import Data.HashMap.Strict (HashMap) +import Data.String +import Data.Text (Text) +import Prelude +import qualified Data.HashMap.Strict as HashMap +import qualified Data.Text as Text + +import Duckling.Dimensions.Types +import Duckling.Numeral.Helpers +import Duckling.Quantity.Helpers +import Duckling.Regex.Types (GroupMatch(..)) +import Duckling.Types +import Duckling.Numeral.Types (NumeralData(..)) +import Duckling.Quantity.Types (QuantityData(..)) +import qualified Duckling.Numeral.Types as TNumeral +import qualified Duckling.Quantity.Types as TQuantity + +-- Our quantities: (name, regex, quantityType). Please make sure to wrap +-- each regex in parentheses, because we extend these regexes later with +-- articles (un/una). +quantities :: [(Text, String, TQuantity.Unit)] +quantities = + [ (" bowls", "(bol(es)?|tazón(es)?|cuencos?|platos? (soperos?)|(hondos?))", TQuantity.Bowl) + , (" cups", "(tazas?)", TQuantity.Cup) + , (" dishes", "(platos?|fuentes?)", TQuantity.Dish) + , (" grams", "(((m(ili)?)|(k(ilo)?))?g(ramo)?s?)", TQuantity.Gram) + , (" ounces", "((onzas?)|oz)", TQuantity.Ounce) + , (" pints", "(pintas?)", TQuantity.Pint) + , (" pounds", "((lb|libra)s?)", TQuantity.Pound) + , (" quarts", "(cuartos? de galón)", TQuantity.Quart) + , (" tablespoons", "(cucharadas? (grande)?)", TQuantity.Tablespoon) + , (" teaspoons", "(cucharaditas?)", TQuantity.Teaspoon) + ] + +opsMap :: HashMap Text (Double -> Double) +opsMap = HashMap.fromList + [ ( "miligram" , (/ 1000)) + , ( "miligramos" , (/ 1000)) + , ( "mg" , (/ 1000)) + , ( "mgs" , (/ 1000)) + , ( "kilogramo" , (* 1000)) + , ( "kilogramos" , (* 1000)) + , ( "kg" , (* 1000)) + , ( "kgs" , (* 1000)) + ] + +ruleNumeralQuantities :: [Rule] +ruleNumeralQuantities = map go quantities + where + go :: (Text, String, TQuantity.Unit) -> Rule + go (nm, regexPattern, u) = Rule + { name = nm + , pattern = [Predicate isPositive, regex regexPattern] + , prod = \case + (Token Numeral nd: + Token RegexMatch (GroupMatch (match:_)): + _) -> Just $ Token Quantity $ quantity u val + where val = getValue opsMap match $ TNumeral.value nd + _ -> Nothing + } + +-- Quantities prefixed by "un" or "una" +ruleAQuantity :: [Rule] +ruleAQuantity = map go quantities + where + go :: (Text, String, TQuantity.Unit) -> Rule + go (nm, regexPattern, u) = Rule + { name = nm + , pattern = [ regex ("una? " ++ regexPattern) ] + , prod = \case + (Token RegexMatch (GroupMatch (match:_)): + _) -> Just $ Token Quantity $ quantity u $ getValue opsMap match 1 + _ -> Nothing + } + +ruleQuantityOfProduct :: Rule +ruleQuantityOfProduct = Rule + { name = " de producto" + , pattern = + [ dimension Quantity + , regex "de (\\w+)" + ] + , prod = \case + (Token Quantity qd:Token RegexMatch (GroupMatch (prdct:_)):_) -> + Just $ Token Quantity $ withProduct (Text.toLower prdct) qd + _ -> Nothing + } + +rulePrecision :: Rule +rulePrecision = Rule + { name = "about|exactly " + , pattern = + [ regex "exactamente|precisamente|a?cerca( de)?|aproximadamente|casi" + , dimension Quantity + ] + , prod = \case + (_:tkn:_) -> Just tkn + _ -> Nothing + } + +ruleIntervalBetweenNumeral :: Rule +ruleIntervalBetweenNumeral = Rule + { name = "between|from and|to " + , pattern = + [ regex "entre|de" + , Predicate isPositive + , regex "a|y" + , Predicate isSimpleQuantity + ] + , prod = \case + (_: + Token Numeral NumeralData{TNumeral.value = from}: + _: + Token Quantity QuantityData{TQuantity.value = Just to + , TQuantity.unit = Just u + , TQuantity.aproduct = Nothing}: + _) | from < to -> + Just $ Token Quantity $ withInterval (from, to) $ unitOnly u + _ -> Nothing + } + +ruleIntervalBetween :: Rule +ruleIntervalBetween = Rule + { name = "between|from to|and " + , pattern = + [ regex "entre/de" + , Predicate isSimpleQuantity + , regex "a/y" + , Predicate isSimpleQuantity + ] + , prod = \case + (_: + Token Quantity QuantityData{TQuantity.value = Just from + , TQuantity.unit = Just u1 + , TQuantity.aproduct = Nothing}: + _: + Token Quantity QuantityData{TQuantity.value = Just to + , TQuantity.unit = Just u2 + , TQuantity.aproduct = Nothing}: + _) | from < to && u1 == u2 -> + Just $ Token Quantity $ withInterval (from, to) $ unitOnly u1 + _ -> Nothing + } + +ruleIntervalNumeralDash :: Rule +ruleIntervalNumeralDash = Rule + { name = " - " + , pattern = + [ Predicate isPositive + , regex "\\-" + , Predicate isSimpleQuantity + ] + , prod = \case + (Token Numeral NumeralData{TNumeral.value = from}: + _: + Token Quantity QuantityData{TQuantity.value = Just to + , TQuantity.unit = Just u + , TQuantity.aproduct = Nothing}: + _) | from < to -> + Just $ Token Quantity $ withInterval (from, to) $ unitOnly u + _ -> Nothing + } + +ruleIntervalDash :: Rule +ruleIntervalDash = Rule + { name = " - " + , pattern = + [ Predicate isSimpleQuantity + , regex "\\-" + , Predicate isSimpleQuantity + ] + , prod = \case + (Token Quantity QuantityData{TQuantity.value = Just from + , TQuantity.unit = Just u1 + , TQuantity.aproduct = Nothing}: + _: + Token Quantity QuantityData{TQuantity.value = Just to + , TQuantity.unit = Just u2 + , TQuantity.aproduct = Nothing}: + _) | from < to && u1 == u2 -> + Just $ Token Quantity $ withInterval (from, to) $ unitOnly u1 + _ -> Nothing + } + +ruleIntervalMax :: Rule +ruleIntervalMax = Rule + { name = "under/below/less/lower/at most/no more than " + , pattern = + [ regex "no m(á|a)s que|menos de|por debajo de|como mucho|como m(á|a)xim(o|a)|a lo sumo|menos (que|de)" + , Predicate isSimpleQuantity + ] + , prod = \case + (_: + Token Quantity QuantityData{TQuantity.value = Just to + , TQuantity.unit = Just u + , TQuantity.aproduct = Nothing}: + _) -> Just $ Token Quantity $ withMax to $ unitOnly u + _ -> Nothing + } + +ruleIntervalMin :: Rule +ruleIntervalMin = Rule + { name = "over/above/exceeding/beyond/at least/more than " + , pattern = + [ regex "(? Just $ Token Quantity $ withMin from $ unitOnly u + _ -> Nothing + } + +ruleQuantityLatent :: Rule +ruleQuantityLatent = Rule + { name = " (latent)" + , pattern = + [ Predicate isPositive + ] + , prod = \case + (Token Numeral NumeralData{TNumeral.value = v}: _) -> + Just $ Token Quantity $ mkLatent $ valueOnly v + _ -> Nothing + } + + +rules :: [Rule] +rules = + [ ruleQuantityOfProduct + , ruleIntervalMin + , ruleIntervalMax + , ruleIntervalBetweenNumeral + , ruleIntervalBetween + , ruleIntervalNumeralDash + , ruleIntervalDash + , rulePrecision + , ruleQuantityLatent + ] + ++ ruleNumeralQuantities + ++ ruleAQuantity diff --git a/Duckling/Rules/ES.hs b/Duckling/Rules/ES.hs index 859cd4f75..bc1c7dad5 100644 --- a/Duckling/Rules/ES.hs +++ b/Duckling/Rules/ES.hs @@ -39,6 +39,7 @@ import qualified Duckling.TimeGrain.ES.Rules as TimeGrain import Duckling.Types import qualified Duckling.Volume.ES.Rules as Volume import qualified Duckling.Duration.ES.Rules as Duration +import qualified Duckling.Quantity.ES.Rules as Quantity defaultRules :: Seal Dimension -> [Rule] defaultRules dim@(Seal Numeral) = @@ -65,7 +66,7 @@ langRules (Seal Email) = [] langRules (Seal Numeral) = Numeral.rules langRules (Seal Ordinal) = Ordinal.rules langRules (Seal PhoneNumber) = [] -langRules (Seal Quantity) = [] +langRules (Seal Quantity) = Quantity.rules langRules (Seal RegexMatch) = [] langRules (Seal Temperature) = Temperature.rules langRules (Seal Time) = Time.rules diff --git a/duckling.cabal b/duckling.cabal index 22a9e09d0..319e9977f 100644 --- a/duckling.cabal +++ b/duckling.cabal @@ -589,6 +589,8 @@ library , Duckling.Quantity.AR.Rules , Duckling.Quantity.EN.Corpus , Duckling.Quantity.EN.Rules + , Duckling.Quantity.ES.Corpus + , Duckling.Quantity.ES.Rules , Duckling.Quantity.FR.Corpus , Duckling.Quantity.FR.Rules , Duckling.Quantity.HR.Corpus @@ -1041,6 +1043,7 @@ test-suite duckling-test -- Quantity , Duckling.Quantity.AR.Tests , Duckling.Quantity.EN.Tests + , Duckling.Quantity.ES.Tests , Duckling.Quantity.FR.Tests , Duckling.Quantity.HR.Tests , Duckling.Quantity.KM.Tests diff --git a/tests/Duckling/Quantity/ES/Tests.hs b/tests/Duckling/Quantity/ES/Tests.hs new file mode 100644 index 000000000..81a8f49a6 --- /dev/null +++ b/tests/Duckling/Quantity/ES/Tests.hs @@ -0,0 +1,22 @@ +-- Copyright (c) 2016-present, Facebook, Inc. +-- All rights reserved. +-- +-- This source code is licensed under the BSD-style license found in the +-- LICENSE file in the root directory of this source tree. + + +module Duckling.Quantity.ES.Tests + ( tests + ) where + +import Data.String +import Test.Tasty + +import Duckling.Dimensions.Types +import Duckling.Quantity.ES.Corpus +import Duckling.Testing.Asserts + +tests :: TestTree +tests = testGroup "ES Tests" + [ makeCorpusTest [Seal Quantity] corpus + ] diff --git a/tests/Duckling/Quantity/Tests.hs b/tests/Duckling/Quantity/Tests.hs index d2c9e63ac..023950a15 100644 --- a/tests/Duckling/Quantity/Tests.hs +++ b/tests/Duckling/Quantity/Tests.hs @@ -10,11 +10,11 @@ module Duckling.Quantity.Tests ) where import Data.String -import Prelude import Test.Tasty import qualified Duckling.Quantity.AR.Tests as AR import qualified Duckling.Quantity.EN.Tests as EN +import qualified Duckling.Quantity.ES.Tests as ES import qualified Duckling.Quantity.FR.Tests as FR import qualified Duckling.Quantity.HR.Tests as HR import qualified Duckling.Quantity.KM.Tests as KM @@ -30,6 +30,7 @@ tests :: TestTree tests = testGroup "Quantity Tests" [ AR.tests , EN.tests + , ES.tests , FR.tests , HR.tests , KM.tests From e7264b55c9091f043bbcd51987c3b60d16d6f539 Mon Sep 17 00:00:00 2001 From: Dmitri Osipov Date: Mon, 9 Nov 2020 11:16:42 -0800 Subject: [PATCH 005/136] adds frequent durations in German (#509) Summary: Found a lacking frequent duration in German and a small typo in the existing one. Pull Request resolved: https://github.com/facebook/duckling/pull/509 Reviewed By: patapizza Differential Revision: D24690104 Pulled By: chessai fbshipit-source-id: b49a7a636abf5b92f2fe7c0d5b2ca2fe64acbaa2 --- Duckling/Duration/DE/Corpus.hs | 97 +++++++++++ Duckling/Duration/DE/Rules.hs | 229 +++++++++++++++++++------- Duckling/Duration/EN/Rules.hs | 3 +- Duckling/Numeral/DE/Rules.hs | 6 +- Duckling/Ranking/Classifiers/DE_XX.hs | 196 ++++++++++++++-------- Duckling/Time/DE/Corpus.hs | 2 +- Duckling/TimeGrain/DE/Rules.hs | 2 +- tests/Duckling/Duration/DE/Tests.hs | 22 +++ 8 files changed, 424 insertions(+), 133 deletions(-) create mode 100644 Duckling/Duration/DE/Corpus.hs create mode 100644 tests/Duckling/Duration/DE/Tests.hs diff --git a/Duckling/Duration/DE/Corpus.hs b/Duckling/Duration/DE/Corpus.hs new file mode 100644 index 000000000..3946d0fb0 --- /dev/null +++ b/Duckling/Duration/DE/Corpus.hs @@ -0,0 +1,97 @@ +-- Copyright (c) 2016-present, Facebook, Inc. +-- All rights reserved. +-- +-- This source code is licensed under the BSD-style license found in the +-- LICENSE file in the root directory of this source tree. + + +{-# LANGUAGE OverloadedStrings #-} + +module Duckling.Duration.DE.Corpus + ( corpus + ) where + +import Prelude +import Data.String + +import Duckling.Duration.Types +import Duckling.Locale +import Duckling.Resolve +import Duckling.Testing.Types +import Duckling.TimeGrain.Types (Grain(..)) + +corpus :: Corpus +corpus = (testContext {locale = makeLocale DE Nothing}, testOptions, allExamples) + +allExamples :: [Example] +allExamples = concat + [ examples (DurationData 1 Second) + [ "ein sekunde" + , "zirka eine sekunde" + ] + , examples (DurationData 30 Minute) + [ "1/2 stunde" + , "1/2stunde" + , "eine halbe stunde" + , "einer halben stunde" + , "halbe stunde" + , "halben stunde" + , "ungefahr einer halben stunde" + ] + , examples (DurationData 15 Minute) + [ "einer Viertelstunde" + , "eine viertelstunde" + , "ViErTelStUnDe" + , "genau viertelstunde" + ] + , examples (DurationData 45 Minute) + [ "3/4 stunde" + , "3/4stunde" + , "eine dreiviertel stunde" + , "einer dreiviertel stunde" + , "dreiviertel stunde" + , "drei viertelstunden" + ] + , examples (DurationData 92 Minute) + [ "92 minuten" + , "zweiundneunzig minuten" + , "eine Stunde und zweiunddreißig Minuten" + , "ein stunde und zweiunddreissig minuten" + ] + , examples (DurationData 30 Day) + [ "30 tage" + , "dreißig tage" + , "dreissig tage" + ] + , examples (DurationData 7 Week) + [ "7 Wochen" + , "sieben wochen" + ] + , examples (DurationData 90 Minute) + [ "1,5 stunden" + , "1,5 stunde" + , "90 min" + , "90min" + ] + , examples (DurationData 75 Minute) + [ "1,25 stunden" + ] + , examples (DurationData 31719604 Second) + [ "1 Jahr, 2 Tage, 3 Stunden und 4 Sekunden" + , "1 Jahr 2 Tage 3 Stunden und 4 Sekunden" + ] + , examples (DurationData 330 Second) + [ "5 und eine halbe Minuten" + , "5,5 min" + ] + , examples (DurationData 330 Minute) + [ "5 und eine halbe stunden" + , "5,5 stunde" + , "exakt 5,5 stunden" + ] + , examples (DurationData 930 Second) + [ "15,5 minuten" + , "15,5 minute" + , "15,5 min" + ] + ] diff --git a/Duckling/Duration/DE/Rules.hs b/Duckling/Duration/DE/Rules.hs index c15290004..405622dc5 100644 --- a/Duckling/Duration/DE/Rules.hs +++ b/Duckling/Duration/DE/Rules.hs @@ -6,131 +6,244 @@ {-# LANGUAGE GADTs #-} +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} module Duckling.Duration.DE.Rules ( rules ) where -import Control.Monad (join) -import qualified Data.Text as Text -import Prelude -import Data.String import Duckling.Dimensions.Types import Duckling.Duration.Helpers -import Duckling.Numeral.Helpers (parseInteger) -import Duckling.Numeral.Types (NumeralData (..)) -import qualified Duckling.Numeral.Types as TNumeral import Duckling.Regex.Types +import Control.Monad (join) +import qualified Data.Text as Text +import Prelude +import Duckling.Numeral.Helpers (parseInteger) +import Duckling.Duration.Types (DurationData(..)) +import qualified Duckling.Duration.Types as TDuration +import Data.String +import Duckling.Numeral.Types (NumeralData(..)) import qualified Duckling.TimeGrain.Types as TG +import qualified Duckling.Numeral.Types as TNumeral import Duckling.Types +ruleQuarterOfAnHour :: Rule +ruleQuarterOfAnHour = Rule + { name = "quarter of an hour" + , pattern = + [ regex "(einer? )?Viertelstunde" + ] + , prod = \_ -> Just $ Token Duration $ duration TG.Minute 15 + } + ruleHalfAnHour :: Rule ruleHalfAnHour = Rule { name = "half an hour" , pattern = - [ regex "(1/2\\s?|(einer )halbe?n? )stunde" + [ regex "(1/2\\s?|(eine )?halbe |(einer )?halben )stunde" ] - , prod = \_ -> Just . Token Duration $ duration TG.Minute 30 + , prod = \_ -> Just $ Token Duration $ duration TG.Minute 30 + } + +ruleThreeQuartersOfAnHour :: Rule +ruleThreeQuartersOfAnHour = Rule + { name = "three-quarters of an hour" + , pattern = + [ regex "3/4\\s?stunde|(einer? )?dreiviertel stunde|drei viertelstunden" + ] + , prod = \_ -> Just $ Token Duration $ duration TG.Minute 45 } ruleFortnight :: Rule ruleFortnight = Rule { name = "fortnight" , pattern = - [ regex "(a|one)? fortnight" + [ regex "zwei Wochen" ] - , prod = \_ -> Just . Token Duration $ duration TG.Day 14 + , prod = \_ -> Just $ Token Duration $ duration TG.Day 14 } -ruleIntegerMoreUnitofduration :: Rule -ruleIntegerMoreUnitofduration = Rule - { name = " more " +rulePrecision :: Rule +rulePrecision = Rule + { name = "about|exactly " , pattern = - [ Predicate isNatural - , regex "mehr|weniger" - , dimension TimeGrain + [ regex "ungef(ä|a)hr|zirka|genau|exakt" + , dimension Duration ] - , prod = \tokens -> case tokens of - (Token Numeral NumeralData{TNumeral.value = v}: - _: - Token TimeGrain grain: - _) -> Just . Token Duration . duration grain $ floor v + , prod = \case + (_:token:_) -> Just token _ -> Nothing } -ruleNumeralnumberHours :: Rule -ruleNumeralnumberHours = Rule - { name = "number.number hours" +ruleCommaNumeralHours :: Rule +ruleCommaNumeralHours = Rule + { name = "number,number hours" , pattern = - [ regex "(\\d+)\\.(\\d+) stunden?" + [ regex "(\\d+),(\\d+)" + , Predicate $ isGrain TG.Hour ] - , prod = \tokens -> case tokens of + , prod = \case (Token RegexMatch (GroupMatch (h:m:_)):_) -> do hh <- parseInteger h mnum <- parseInteger m let mden = 10 ^ Text.length m - Just . Token Duration $ minutesFromHourMixedFraction hh mnum mden + Just $ Token Duration $ minutesFromHourMixedFraction hh mnum mden _ -> Nothing } -ruleIntegerAndAnHalfHours :: Rule -ruleIntegerAndAnHalfHours = Rule - { name = " and an half hours" +ruleCommaNumeralMinutes :: Rule +ruleCommaNumeralMinutes = Rule + { name = "number,number minutes" + , pattern = + [ regex "(\\d+),(\\d+)" + , Predicate $ isGrain TG.Minute + ] + , prod = \case + (Token RegexMatch (GroupMatch (m:s:_)):_) -> do + mm <- parseInteger m + ss <- parseInteger s + let sden = 10 ^ Text.length s + Just $ Token Duration $ secondsFromHourMixedFraction mm ss sden + _ -> Nothing + } + +ruleAndHalfHour :: Rule +ruleAndHalfHour = Rule + { name = " and a half hour" , pattern = [ Predicate isNatural - , regex "ein ?halb stunden?" + , regex "(und )?(ein(en?)? )?halb(en?)?" + , Predicate $ isGrain TG.Hour ] - , prod = \tokens -> case tokens of + , prod = \case (Token Numeral NumeralData{TNumeral.value = v}:_) -> - Just . Token Duration . duration TG.Minute $ 30 + 60 * floor v + Just $ Token Duration $ duration TG.Minute $ 30 + 60 * floor v _ -> Nothing } -ruleAUnitofduration :: Rule -ruleAUnitofduration = Rule +ruleAndHalfMinute :: Rule +ruleAndHalfMinute = Rule + { name = " and a half minutes" + , pattern = + [ Predicate isNatural + , regex "(und )?(ein(en?)? ?)?halb(en?)?" + , Predicate $ isGrain TG.Minute + ] + , prod = \case + (Token Numeral NumeralData{TNumeral.value = v}:_) -> + Just $ Token Duration $ duration TG.Second $ 30 + 60 * floor v + _ -> Nothing + } + +ruleArticle :: Rule +ruleArticle = Rule { name = "a " , pattern = - [ regex "eine?(r|n)?" + [ regex "ein(en?)?" + , dimension TimeGrain + ] + , prod = \case + (_:Token TimeGrain grain:_) -> Just $ Token Duration $ duration grain 1 + _ -> Nothing + } + +ruleHalfTimeGrain :: Rule +ruleHalfTimeGrain = Rule + { name = "half a " + , pattern = + [ regex "(ein(en)?)?(1/2|halbe?)" , dimension TimeGrain ] - , prod = \tokens -> case tokens of - (_:Token TimeGrain grain:_) -> Just . Token Duration $ duration grain 1 + , prod = \case + (_:Token TimeGrain grain:_) -> Token Duration <$> nPlusOneHalf grain 0 _ -> Nothing } -ruleAboutDuration :: Rule -ruleAboutDuration = Rule - { name = "about " +ruleCompositeDurationCommasAnd :: Rule +ruleCompositeDurationCommasAnd = Rule + { name = "composite (with ,/and)" , pattern = - [ regex "ungefähr|zirka" + [ Predicate isNatural + , dimension TimeGrain + , regex ",|und" , dimension Duration ] - , prod = \tokens -> case tokens of - (_:token:_) -> Just token + , prod = \case + (Token Numeral NumeralData{TNumeral.value = v}: + Token TimeGrain g: + _: + Token Duration dd@DurationData{TDuration.grain = dg}: + _) | g > dg -> Just $ Token Duration $ duration g (floor v) <> dd _ -> Nothing } -ruleExactlyDuration :: Rule -ruleExactlyDuration = Rule - { name = "exactly " +ruleCompositeDuration :: Rule +ruleCompositeDuration = Rule + { name = "composite " , pattern = - [ regex "genau|exakt" + [ Predicate isNatural + , dimension TimeGrain , dimension Duration ] - , prod = \tokens -> case tokens of - (_:token:_) -> Just token + , prod = \case + (Token Numeral NumeralData{TNumeral.value = v}: + Token TimeGrain g: + Token Duration dd@DurationData{TDuration.grain = dg}: + _) | g > dg -> Just $ Token Duration $ duration g (floor v) <> dd + _ -> Nothing + } + +ruleCompositeDurationAnd :: Rule +ruleCompositeDurationAnd = Rule + { name = "composite and " + , pattern = + [ dimension Duration + , regex ",|und" + , dimension Duration + ] + , prod = \case + (Token Duration DurationData{TDuration.value = v, TDuration.grain = g}: + _: + Token Duration dd@DurationData{TDuration.grain = dg}: + _) | g > dg -> Just $ Token Duration $ duration g v <> dd + _ -> Nothing + } + +ruleHoursAndMinutes :: Rule +ruleHoursAndMinutes = Rule + { name = " hour and " + , pattern = + [ Predicate isNatural + , regex "(ein(en?) )?stunden?( und)?" + , Predicate isNatural + , Predicate $ isGrain TG.Minute + ] + , prod = \case + (Token Numeral h: + _: + Token Numeral m: + _) -> Just $ Token Duration $ duration TG.Minute $ + floor (TNumeral.value m) + 60 * floor (TNumeral.value h) _ -> Nothing } rules :: [Rule] rules = - [ ruleAUnitofduration - , ruleAboutDuration - , ruleExactlyDuration - , ruleFortnight + [ ruleQuarterOfAnHour , ruleHalfAnHour - , ruleIntegerAndAnHalfHours - , ruleIntegerMoreUnitofduration - , ruleNumeralnumberHours + , ruleThreeQuartersOfAnHour + , rulePrecision + , ruleCommaNumeralHours + , ruleCommaNumeralMinutes + , ruleFortnight + , ruleAndHalfHour + , ruleAndHalfMinute + , ruleArticle + , ruleHalfTimeGrain + , ruleCompositeDurationCommasAnd + , ruleCompositeDuration + , ruleCompositeDurationAnd + , ruleHoursAndMinutes + , ruleAndHalfMinute ] diff --git a/Duckling/Duration/EN/Rules.hs b/Duckling/Duration/EN/Rules.hs index 44309a34f..265b04fbd 100644 --- a/Duckling/Duration/EN/Rules.hs +++ b/Duckling/Duration/EN/Rules.hs @@ -15,14 +15,13 @@ module Duckling.Duration.EN.Rules ) where import Data.Semigroup ((<>)) -import Data.String import Prelude import qualified Data.Text as Text import Duckling.Dimensions.Types import Duckling.Duration.Helpers import Duckling.Duration.Types (DurationData(..)) -import Duckling.Numeral.Helpers (parseInt, parseInteger) +import Duckling.Numeral.Helpers (parseInteger) import Duckling.Numeral.Types (NumeralData(..)) import Duckling.Regex.Types import Duckling.Types diff --git a/Duckling/Numeral/DE/Rules.hs b/Duckling/Numeral/DE/Rules.hs index fd8c0f79b..e551aeb3e 100644 --- a/Duckling/Numeral/DE/Rules.hs +++ b/Duckling/Numeral/DE/Rules.hs @@ -89,7 +89,7 @@ ruleInteger3 :: Rule ruleInteger3 = Rule { name = "integer ([2-9][1-9])" , pattern = - [ regex "(ein|zwei|drei|vier|fünf|sechs|sieben|acht|neun)und(zwanzig|dreissig|vierzig|fünfzig|sechzig|siebzig|achtzig|neunzig)" + [ regex "(ein|zwei|drei|vier|fünf|sechs|sieben|acht|neun)und(zwanzig|dreissig|dreißig|vierzig|fünfzig|sechzig|siebzig|achtzig|neunzig)" ] , prod = \tokens -> case tokens of (Token RegexMatch (GroupMatch (m1:m2:_)):_) -> do @@ -107,6 +107,7 @@ ruleInteger3 = Rule v2 <- case Text.toLower m2 of "zwanzig" -> Just 20 "dreissig" -> Just 30 + "dreißig" -> Just 30 "vierzig" -> Just 40 "fünfzig" -> Just 50 "sechzig" -> Just 60 @@ -266,6 +267,7 @@ tensMap :: HashMap Text Integer tensMap = HashMap.fromList [ ( "zwanzig" , 20 ) , ( "dreissig", 30 ) + , ( "dreißig" , 30 ) , ( "vierzig" , 40 ) , ( "fünfzig" , 50 ) , ( "sechzig" , 60 ) @@ -279,7 +281,7 @@ ruleInteger2 :: Rule ruleInteger2 = Rule { name = "integer (20..90)" , pattern = - [ regex "(zwanzig|dreissig|vierzig|fünfzig|sechzig|siebzig|achtzig|neunzig)" + [ regex "(zwanzig|dreissig|dreißig|vierzig|fünfzig|sechzig|siebzig|achtzig|neunzig)" ] , prod = \tokens -> case tokens of (Token RegexMatch (GroupMatch (match:_)):_) -> diff --git a/Duckling/Ranking/Classifiers/DE_XX.hs b/Duckling/Ranking/Classifiers/DE_XX.hs index a65ad8467..aa08f62be 100644 --- a/Duckling/Ranking/Classifiers/DE_XX.hs +++ b/Duckling/Ranking/Classifiers/DE_XX.hs @@ -224,12 +224,12 @@ classifiers likelihoods = HashMap.fromList [], n = 0}}), ("mm/dd", Classifier{okData = - ClassData{prior = -0.40546510810816444, + ClassData{prior = -0.37729423114146804, unseen = -3.258096538021482, likelihoods = HashMap.fromList [("", 0.0)], n = 24}, koData = - ClassData{prior = -1.0986122886681098, unseen = -2.639057329615259, - likelihoods = HashMap.fromList [("", 0.0)], n = 12}}), + ClassData{prior = -1.157452788691043, unseen = -2.5649493574615367, + likelihoods = HashMap.fromList [("", 0.0)], n = 11}}), ("at ", Classifier{okData = ClassData{prior = -0.137201121513485, unseen = -4.3694478524670215, @@ -474,13 +474,13 @@ classifiers likelihoods = HashMap.fromList [], n = 0}}), ("hour (grain)", Classifier{okData = - ClassData{prior = -0.15415067982725836, - unseen = -2.0794415416798357, - likelihoods = HashMap.fromList [("", 0.0)], n = 6}, + ClassData{prior = -0.2231435513142097, + unseen = -2.3025850929940455, + likelihoods = HashMap.fromList [("", 0.0)], n = 8}, koData = - ClassData{prior = -1.9459101490553135, - unseen = -1.0986122886681098, - likelihoods = HashMap.fromList [("", 0.0)], n = 1}}), + ClassData{prior = -1.6094379124341003, + unseen = -1.3862943611198906, + likelihoods = HashMap.fromList [("", 0.0)], n = 2}}), ("Gr\252ndonnerstag", Classifier{okData = ClassData{prior = 0.0, unseen = -1.9459101490553135, @@ -519,6 +519,17 @@ classifiers [("ordinal (digits)quarter (grain)", -0.916290731874155), ("quarter", -0.916290731874155)], n = 1}}), + ("a ", + Classifier{okData = + ClassData{prior = -infinity, unseen = -1.0986122886681098, + likelihoods = HashMap.fromList [], n = 0}, + koData = + ClassData{prior = 0.0, unseen = -1.6094379124341003, + likelihoods = + HashMap.fromList + [("half a ", -0.6931471805599453), + ("minute", -0.6931471805599453)], + n = 1}}), ("Mai", Classifier{okData = ClassData{prior = 0.0, unseen = -1.0986122886681098, @@ -533,6 +544,23 @@ classifiers koData = ClassData{prior = -infinity, unseen = -0.6931471805599453, likelihoods = HashMap.fromList [], n = 0}}), + (" and a half hour", + Classifier{okData = + ClassData{prior = -1.3862943611198906, + unseen = -1.6094379124341003, + likelihoods = + HashMap.fromList + [("integer (0..19)hour (grain)", -0.6931471805599453), + ("hour", -0.6931471805599453)], + n = 1}, + koData = + ClassData{prior = -0.2876820724517809, + unseen = -2.1972245773362196, + likelihoods = + HashMap.fromList + [("integer (0..19)hour (grain)", -0.6931471805599453), + ("hour", -0.6931471805599453)], + n = 3}}), ("intersect", Classifier{okData = ClassData{prior = -0.11795690334351183, @@ -947,13 +975,6 @@ classifiers [("week", -1.6094379124341003), ("week (grain)", -1.6094379124341003)], n = 1}}), - ("number.number hours", - Classifier{okData = - ClassData{prior = 0.0, unseen = -1.0986122886681098, - likelihoods = HashMap.fromList [("", 0.0)], n = 1}, - koData = - ClassData{prior = -infinity, unseen = -0.6931471805599453, - likelihoods = HashMap.fromList [], n = 0}}), ("Dreifaltigkeitssonntag", Classifier{okData = ClassData{prior = 0.0, unseen = -1.791759469228055, @@ -1134,6 +1155,17 @@ classifiers ("daymonth", -1.0986122886681098), ("ordinal (1..31)DienstagSeptember", -1.791759469228055)], n = 3}}), + ("half a ", + Classifier{okData = + ClassData{prior = -infinity, unseen = -1.0986122886681098, + likelihoods = HashMap.fromList [], n = 0}, + koData = + ClassData{prior = 0.0, unseen = -1.6094379124341003, + likelihoods = + HashMap.fromList + [("hour (grain)", -0.6931471805599453), + ("hour", -0.6931471805599453)], + n = 1}}), ("the (non ordinal)", Classifier{okData = ClassData{prior = -infinity, unseen = -0.6931471805599453, @@ -1619,6 +1651,17 @@ classifiers koData = ClassData{prior = -infinity, unseen = -0.6931471805599453, likelihoods = HashMap.fromList [], n = 0}}), + ("number,number hours", + Classifier{okData = + ClassData{prior = 0.0, unseen = -1.6094379124341003, + likelihoods = + HashMap.fromList + [("hour (grain)", -0.6931471805599453), + ("hour", -0.6931471805599453)], + n = 1}, + koData = + ClassData{prior = -infinity, unseen = -1.0986122886681098, + likelihoods = HashMap.fromList [], n = 0}}), ("Guru Gobind Singh Jayanti", Classifier{okData = ClassData{prior = 0.0, unseen = -1.6094379124341003, @@ -1711,21 +1754,14 @@ classifiers likelihoods = HashMap.fromList [], n = 0}}), ("a ", Classifier{okData = - ClassData{prior = 0.0, unseen = -3.1354942159291497, + ClassData{prior = 0.0, unseen = -1.6094379124341003, likelihoods = HashMap.fromList - [("week", -1.9924301646902063), - ("hour (grain)", -2.3978952727983707), - ("year (grain)", -2.3978952727983707), - ("second", -2.3978952727983707), - ("week (grain)", -1.9924301646902063), - ("minute (grain)", -2.3978952727983707), - ("year", -2.3978952727983707), - ("second (grain)", -2.3978952727983707), - ("hour", -2.3978952727983707), ("minute", -2.3978952727983707)], - n = 6}, + [("year (grain)", -0.6931471805599453), + ("year", -0.6931471805599453)], + n = 1}, koData = - ClassData{prior = -infinity, unseen = -2.3978952727983707, + ClassData{prior = -infinity, unseen = -1.0986122886681098, likelihoods = HashMap.fromList [], n = 0}}), ("Dhanteras", Classifier{okData = @@ -1939,18 +1975,23 @@ classifiers likelihoods = HashMap.fromList [("", 0.0)], n = 3}}), (" ago", Classifier{okData = - ClassData{prior = 0.0, unseen = -3.1354942159291497, + ClassData{prior = -0.13353139262452263, + unseen = -3.044522437723423, likelihoods = HashMap.fromList - [("week", -1.4816045409242156), ("day", -1.9924301646902063), - ("year", -2.3978952727983707), - (" ", -1.0116009116784799), - ("a ", -2.3978952727983707), - ("month", -2.3978952727983707)], - n = 8}, + [("week", -1.6094379124341003), ("day", -1.8971199848858813), + ("year", -2.3025850929940455), + (" ", -0.916290731874155), + ("month", -2.3025850929940455)], + n = 7}, koData = - ClassData{prior = -infinity, unseen = -1.9459101490553135, - likelihoods = HashMap.fromList [], n = 0}}), + ClassData{prior = -2.0794415416798357, + unseen = -2.1972245773362196, + likelihoods = + HashMap.fromList + [("day", -1.3862943611198906), + ("fortnight", -1.3862943611198906)], + n = 1}}), ("last