1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225openCommonopenModule_typesmodulePosition:sigtypetvalline:t->intvalcolumn:t->intvalstart:tvalnext:char->t->tvalnext_line:t->tvalnext_column:t->tend=structtypet={line:int;column:int}letline(p:t):int=p.lineletcolumn(p:t):int=p.columnletstart:t={line=0;column=0}letnext_column(p:t):t={pwithcolumn=p.column+1}letnext_line(p:t):t={line=p.line+1;column=0;}letnext(c:char)(p:t):t=ifc='\n'thennext_linepelsenext_columnpendmoduletypeCONTEXT=sigtypettypemsgvalmessage:t->msgvalposition:t->Position.tvalline:t->intvalcolumn:t->intendmoduleLocated=structtype'at={start:Position.t;value:'a;end_:Position.t}letmakestartvalueend_={start;value;end_}letmap(f:'a->'b)(l:'at):'bt={lwithvalue=fl.value}letuse(l:'at)(f:Position.t->'a->Position.t->'b):'b=fl.startl.valuel.end_letvalue(l:'at):'a=l.valueletstart(l:'at):Position.t=l.startletend_(l:'at)=l.end_letrange(l:'at):Position.t*Position.t=l.start,l.end_endmoduleIndent=structtypet={lb:int;(* lower bound of the indentation set *)ub:intoption;(* upper bound of the indentation set *)abs:bool;(* absolute alignment *)strict:bool;(* default token indentation
true: token indentation > parent
false: token indentation >= parent *)}letinitial:t={lb=0;ub=None;abs=false;strict=false}letbounds(ind:t):int*intoption=ind.lb,ind.ubletis_allowed_token_position(pos:int)(ind:t):bool=ifind.absthen(* The token position must be in the set of the allowed indentations
of the parent. *)ind.lb<=pos&&matchind.ubwith|None->true|Someub->pos<=ubelse(* The token must be strictly or nonstrictly indented relative to the
parent. *)letincr=ifind.strictthen1else0inind.lb+incr<=posletis_offside(col:int)(ind:t):bool=not(is_allowed_token_positioncolind)lettoken(pos:int)(ind:t):t=ifind.absthen(* It is the first token of an absolutely aligned parent. *){indwithlb=pos;ub=Somepos;abs=false}else(* Indentation of the parent is at most the indentation of the token
(strict = false) or the indentation of the token - 1 (strict =
true). *)letpos=ifind.strictthenpos-1elseposinmatchind.ubwith|Someubwhenub<=pos->ind|_->{indwithub=Somepos}letabsolute(ind:t):t={indwithabs=true}letstart_indented(strict:bool)(ind:t):t=ifind.absthenindelseletincr=ifstrictthen1else0in{indwithlb=ind.lb+incr;ub=None}letend_indented(strict:bool)(ind0:t)(ind:t):t=ifind0.absthenindelsematchind.ubwith|None->ind0|Someub->letincr=ifstrictthen1else0inassert(incr<=ub);{ind0withub=matchind0.ubwith|None->Some(ub-1)|Someub0->Some(minub0(ub-1))}endmoduleContext(Msg:ANY)=structtypemsg=Msg.ttypet={pos:Position.t;msg:Msg.t;}letmakeposmsg:t={pos;msg}letmessage(c:t)=c.msgletposition(c:t)=c.posletline(c:t)=Position.linec.posletcolumn(c:t)=Position.columnc.posendmoduleState(User:ANY)(Context_msg:ANY)=structmoduleContext=Context(Context_msg)typecontext=Context.ttypet={pos:Position.t;indent:Indent.t;user:User.t;contexts:contextlist}letmakeposuser={pos;user;indent=Indent.initial;contexts=[]}letposition(s:t):Position.t=s.posletline(s:t):int=Position.lines.posletcolumn(s:t):int=Position.columns.posletnext(c:char)(s:t):t={swithpos=Position.nextcs.pos;indent=Indent.token(Position.columns.pos)s.indent}letbounds(s:t):int*intoption=Indent.boundss.indentletis_offside(s:t):bool=Indent.is_offside(Position.columns.pos)s.indentletabsolute(s:t):t={swithindent=Indent.absolutes.indent}letstart_detached(s:t):t={swithindent=Indent.initial}letend_detached(s0:t)(s:t):t={swithindent=s0.indent}letstart_indented(strict:bool)(s:t):t={swithindent=Indent.start_indentedstricts.indent}letend_indented(strict:bool)(s0:t)(s:t):t={swithindent=Indent.end_indentedstricts0.indents.indent}letuser(s:t):User.t=s.userletupdate(f:User.t->User.t)(s:t)={swithuser=fs.user}letpush_context(msg:Context_msg.t)(s:t):t={swithcontexts=Context.makes.posmsg::s.contexts}letpop_context(s:t):t=matchs.contextswith|[]->assertfalse(* Illegal call *)|_::contexts->{swithcontexts}endmoduletypePARSER=sigtypeparservalneeds_more:parser->boolvalhas_ended:parser->boolvalhas_succeeded:parser->boolvalposition:parser->Position.tvalline:parser->intvalcolumn:parser->intvalput_char:parser->char->parservalput_end:parser->parserendmoduletypeCOMBINATORS=sigincludeGeneric_parser.COMBINATORSvalget_position:(Position.t)tvallocated:'at->'aLocated.tttypestatevalget_state:statetvalupdate:(state->state)->unittvalabsolute:'at->'atvalindented:bool->'at->'atvaldetached:'at->'atvalget_bounds:(int*intoption)tvalone_or_more_aligned:'at->'alisttvalzero_or_more_aligned:'at->'alisttvalskip_one_or_more_aligned:'at->inttvalskip_zero_or_more_aligned:'at->intttypecontextvalin_context:context->'at->'atendmoduleAdvanced(User:ANY)(Final:ANY)(Expect:ANY)(Semantic:ANY)(Context_msg:ANY)=structtypestate=User.ttypecontext=Context_msg.tmoduleToken=structtypet=charoptionendmoduleContext=Context(Context_msg)moduleState=State(User)(Context_msg)moduleBasic=Generic_parser.Make(Token)(State)(Expect)(Semantic)(Final)includeBasicletstate(p:parser):User.t=State.user(Basic.statep)letposition(p:parser):Position.t=State.position(Basic.statep)letline(p:parser):int=State.line(Basic.statep)letcolumn(p:parser):int=State.column(Basic.statep)letget_state:User.tt=Basic.get>>=funst->return(State.userst)letupdate(f:User.t->User.t):unitt=Basic.update(State.updatef)letget_position:(Position.t)t=Basic.get>>=funst->return(State.positionst)letfail(e:Semantic.t):'at=Basic.failelettoken(f:State.t->char->('a,Expect.t)result)(e:State.t->Expect.t)(* generate expectation error in case there
is no character or offside *):'at=Basic.token(funstt->matchtwith|None->Error(est)|Somec->ifState.is_offsidestthenError(est)elsematchfstcwith|Oka->Ok(a,State.nextcst)|Errore->Errore)letbacktrackable(p:'at)(e:Expect.t):'at=Basic.backtrackablepe(* Character Combinators *)letexpect(p:char->bool)(e:Expect.t):chart=token(fun_c->ifpcthenOkcelseErrore)(fun_->e)letexpect_end(e:Expect.t):unitt=Basic.token(funstt->matchtwith|None->Ok((),st)|Some_->Errore)letchar(c:char)(e:Expect.t):unitt=token(fun_d->ifc=dthenOk()elseErrore)(fun_->e)letone_of_chars(str:string)(e:Expect.t):unitt=token(fun_c->ifString.find(fund->c=d)0str=String.lengthstrthenErroreelseOk())(fun_->e)letspace(e:Expect.t):unitt=char' 'eletstring(str:string)(msg:int->Expect.t):unitt=letlen=String.lengthstrinletrecparsei=ifi=lenthenreturn()elsecharstr.[i](msgi)>>=fun_->parse(i+1)inparse0letword(start:char->bool)(inner:char->bool)(e:Expect.t):stringt=letmoduleArr=Segmented_arrayinletrecrestarr=(expectinnere>>=func->rest(Arr.pushcarr))<|>returnarrinexpectstarte>>=func->mapArr.to_string(rest(Arr.singletonc))letwhitespace_char(e:Expect.t):chart=expect(func->c=' '||c='\n'||c='\t')eletwhitespace(e:Expect.t):intt=skip_zero_or_more(map(fun_->())(whitespace_chare))letletter(e:Expect.t):chart=expectChar.is_lettereletdigit(e:Expect.t):chart=expectChar.is_digite(* Context *)letin_context(msg:Context_msg.t)(p:'at):'at=Basic.update(State.push_contextmsg)>>=fun_->p>>=funa->Basic.updateState.pop_context>>=fun_->returna(* Located *)letlocated(p:'at):'aLocated.tt=Basic.get>>=funst1->p>>=funa->Basic.get>>=funst2->return@@Located.make(State.positionst1)a(State.positionst2)(* Indentation combinators *)letabsolute(p:'at):'at=Basic.updateState.absolute>>=fun_->pletindented(strict:bool)(p:'at):'at=Basic.get_and_update(State.start_indentedstrict)>>=funst->p>>=funa->Basic.update(State.end_indentedstrictst)>>=fun_->returnaletdetached(p:'at):'at=Basic.get_and_updateState.start_detached>>=funst->p>>=funa->Basic.update(State.end_detachedst)>>=fun_->returnaletget_bounds:(int*intoption)t=mapState.boundsBasic.getletone_or_more_aligned(p:'at):'alistt=absolute(one_or_more(absolutep))letzero_or_more_aligned(p:'at):'alistt=absolute(zero_or_more(absolutep))letskip_one_or_more_aligned(p:'at):intt=absolute(skip_one_or_more(absolutep))letskip_zero_or_more_aligned(p:'at):intt=absolute(skip_zero_or_more(absolutep))(* General functions *)letput_char(p:parser)(c:char):parser=assert(needs_morep);Basic.put_tokenp(Somec)letput_end(p:parser):parser=assert(needs_morep);Basic.put_tokenpNoneletmake(p:finalt)(user:User.t):parser=Basic.make_parser(State.makePosition.startuser)pletrun(pc:finalt)(user:User.t)(s:string):parser=letp=ref(makepcuser)inleti=ref0andlen=String.lengthsinwhile!i<>len&&needs_more!pdop:=put_char!ps.[!i];i:=!i+1done;ifneeds_more!pthenp:=put_end!p;!pletlookahead_string(p:parser):string=assert(has_endedp);"["^String.concat"; "(List.map(funo->matchowith|None->"None"|Somec->"Some "^"'"^String.onec^"'")(lookaheadp))^"]"letresult_string(p:parser)(f:final->string):string=assert(has_endedp);matchresultpwith|Somea->"Some "^fa|None->"None"endmoduleSimple(Final:ANY)=structmoduleAdvanced=Advanced(Unit)(Final)(String)(String)(String)includeAdvancedletexpect_end:unitt=Advanced.expect_end"end"letchar(c:char):unitt=Advanced.charc@@"'"^String.onec^"'"letone_of_chars(str:string)(msg:string):unitt=Advanced.one_of_charsstrmsgletspace:unitt=Advanced.space"space"letstring(str:string):unitt=Advanced.stringstr(funi->"'"^String.onestr.[i]^"'")letwhitespace_char:chart=Advanced.whitespace_char"whitespace"letwhitespace:intt=Advanced.whitespace"whitespace"letletter:chart=Advanced.letter"letter"letdigit:chart=Advanced.digit"digit"letmake(p:finalt):parser=Advanced.makep()letrun(pc:finalt)(s:string):parser=Advanced.runpc()sletresult_string(p:parser)(f:final->string):string=Advanced.result_stringpfend(* ********** *)(* Unit Tests *)(* ********** *)moduleSimple_test(F:ANY)=structincludeSimple(F)letone_expect(str:string):Error.t=Error.make_expectations[str]endmoduleCP=Simple_test(Char)moduleUP=Simple_test(Unit)moduleIP=Simple_test(Int)moduleSP=Simple_test(String)let%test_=letopenCPinletp=runletter"a"inhas_endedp&&resultp=Some'a'&&columnp=1&&lookaheadp=[]let%test_=letopenCPinletp=run(returnidentity|=letter|.expect_end)"a"inhas_endedp&&resultp=Some'a'&&columnp=1&&lookaheadp=[]moduleCtx=Context(String)let%test_=letopenCPinletp=runletter"1"inhas_endedp&&resultp=None&&errorp=Error.make_expectations["letter"]&&columnp=0&&lookaheadp=[Some'1']let%test_=letopenUPinletp=run(char'a')"z"inhas_endedp&&resultp=None&&errorp=one_expect"'a'"&&columnp=0&&lookaheadp=[Some'z']let%test_=letopenUPinletp=run(char'a'|.expect_end)"ab"inhas_endedp&&resultp=None&&errorp=one_expect"end"&&columnp=1&&lookaheadp=[Some'b']let%test_=letopenUPinletp=run(char'a')"a"inhas_endedp&&resultp=Some()&&columnp=1&&lookaheadp=[]let%test_=letopenUPinletp=run(char'a'|.char'b'|.expect_end)"ab"inhas_endedp&&resultp=Some()&&columnp=2&&lookaheadp=[]let%test_=letopenUPinletp=run(char'a'|.char'b')"a"inhas_endedp&&resultp=None&&errorp=one_expect"'b'"&&columnp=1&&lookaheadp=[None]let%test_=letopenUPinletp=run(char'a'>>=fun_->char'b')"ab"inhas_endedp&&resultp=Some()&&columnp=2&&lookaheadp=[](* Test [optional] *)(* *************** *)let%test_=letopenUPinletp=run(map(fun_->())(char'a'|>optional))"a"inhas_endedp&&columnp=1&&lookaheadp=[]let%test_=letopenUPinletp=run(map(fun_->())(char'a'|>optional))"b"inhas_endedp&&columnp=0&&lookaheadp=[Some'b'](* Test nested parenthesis *)(* *********************** *)letparens:unitUP.t=letopenUPinletrecpars():unitt=(consumer(char'(')>>=pars>>=fun_->char')'>>=pars)<|>return()inpars()letnesting:intIP.t=letopenIPinletrecpars():intt=(consumer(char'(')>>=pars>>=funn->char')'>>=pars>>=funm->return(max(n+1)m))<|>return0inpars()let%test_=letopenUPinletp=runparens"(())()"inhas_endedp&&columnp=6&&lookaheadp=[None]let%test_=letopenUPinletp=runparens"(())("inhas_endedp&&columnp=5&&resultp=None&&errorp=Error.make_expectations["'('";"')'"]&&lookaheadp=[None]let%test_=letopenUPinletp=runparens")"inhas_endedp&&columnp=0&&resultp=Some()&&lookaheadp=[Some')']let%test_=letopenIPinletp=runnesting"(())()"inhas_endedp&&resultp=Some2&&lookaheadp=[None]let%test_=letopenIPinletp=runnesting"(()(()))"inhas_endedp&&resultp=Some3&&lookaheadp=[None](* String parser *)(* ************* *)let%test_=letopenUPinletp=run(string"abcd")"abcd"inhas_endedp&&columnp=4&&resultp=Some()&&lookaheadp=[]let%test_=letopenUPinletp=run(string"(a)"<|>string"(b)")"(b)"inhas_endedp&&columnp=1&&resultp=None&&errorp=one_expect"'a'"&&lookaheadp=[Some'b'](* Backtrackable *)(* ************* *)let%test_=letopenUPinletstr="(a)"inletp=run(backtrackable(stringstr)str)"(a"inhas_endedp&&linep=0&&columnp=0&&resultp=None&&errorp=one_expectstr&&lookaheadp=[Some'(';Some'a';None]let%test_=letopenUPinletp=run(backtrackable(string"(a)")"(a)"<|>string"(b)")"(b)"inhas_endedp&&columnp=3&&resultp=Some()&&lookaheadp=[]let%test_=letopenUPinletp=run((backtrackable(string"(a)")"(a)"<|>string"(b)")|.expect_end)"(b)"inhas_endedp&&columnp=3&&resultp=Some()&&lookaheadp=[](* Not Followed By *)(* *************** *)let%test_=letopenUPinletp=run(not_followed_by(string"abc")"not 'abc'")"abc"inhas_endedp&&columnp=0&&resultp=None&&lookaheadp=[Some'a';Some'b';Some'c']let%test_=letopenUPinletp=run(not_followed_by(string"abc")"not 'abc'")"abx"inhas_endedp&&columnp=0&&resultp=Some()&&lookaheadp=[Some'a';Some'b';Some'x']let%test_=letopenUPinletp=run(backtrackable(char':'|.not_followed_by(char'=')"not '='")":")":="inhas_endedp&&columnp=0&&resultp=None&&lookaheadp=[Some':';Some'=']let%test_=letopenUPinletp=run(backtrackable(char':'|.not_followed_by(char'=')"not '='")":")":"inhas_endedp&&columnp=1&&resultp=Some()&&lookaheadp=[None](* Parser Pipelines *)(* **************** *)let%test_=letmoduleSP=Simple(String)inletopenSPinletp=run(return(func1c2c3->String.onec1^String.onec2^String.onec3)|=letter|.letter|=digit|=letter|.digit)"ab1d0"inhas_endedp&&resultp=Some"a1d"&&columnp=5&&lookaheadp=[](* Indentation sensitivity *)(* *********************** *)moduleIndent_parser=structmoduleP=Simple(Unit)includePletwhite_space:intt=detachedP.whitespaceletletter_ws:chart=returnidentity|=letter|.white_spaceletresult_string(p:parser):string=result_stringp(fun_->"()")letprint(p:parser)(str:string):unit=letopenPrintfinprintf"string <%s>\n"(String.escapedstr);printf"line %d, column %d\n"(linep)(columnp);printf"%s\n"(result_stringp);printf"lookahead %s\n\n"(lookahead_stringp)let_=print(* to avoid warning of unused 'print' *)endlet%test_=letopenIndent_parserinletstr="a\nb"inletp=run(return()|.letter_ws|.(indentedtrueletter_ws)|.expect_end)strin(*print p str;*)has_endedp&&has_failedp&&lookaheadp=[Some'b']&&linep=1&&columnp=0let%test_=letopenIndent_parserinletstr="a\n b\nc"inletp=run(return()|.letter_ws|.(indentedtrueletter_ws)|.letter_ws|.expect_end)strin(*print p str;*)has_endedp&&has_succeededp&&lookaheadp=[]&&linep=2&&columnp=1let%test_=letopenIndent_parserinletstr="a\n b c\n d\n e\nz"inletp=run(return()|.letter_ws|.(indentedtrue(skip_one_or_moreletter_ws))|.letter_ws|.expect_end)strin(*print p str;*)has_endedp&&has_succeededp&&lookaheadp=[]&&linep=4&&columnp=1let%test_=letopenIndent_parserinletstr=" a\n b\n "inletp=run(return()|.white_space|.absolute(return()|.absolute(letter_ws)|.absolute(letter_ws))|.expect_end)strin(*print p str;*)has_endedp&&has_succeededp&&lookaheadp=[]&&linep=2&&columnp=2let%test_=letopenIndent_parserinletstr=" a\n b"inletp=run(return()|.white_space|.absolute(return()|.absolute(letter_ws)|.absolute(letter_ws))|.expect_end)strin(*print p str;*)has_endedp&&has_failedp&&lookaheadp=[Some'b']&&linep=1&&columnp=2let%test_=letopenIndent_parserinletstr="a\nb\n c\n d"inletp=run(return()|.white_space|.absolute(return()|.absolute(letter_ws)|.absolute(letter_ws)|.indentedtrue(absolute(return()|.absolute(letter_ws)|.absolute(letter_ws))))|.expect_end)strin(*print p str;*)has_endedp&&has_succeededp&&lookaheadp=[]&&linep=3&&columnp=2let%test_=letopenIndent_parserinletstr="a\nb\n c\nd"inletp=run(return()|.white_space|.absolute(return()|.absolute(letter_ws)|.absolute(letter_ws)|.indentedtrue(absolute(return()|.absolute(letter_ws)|.absolute(letter_ws))))|.expect_end)strin(*print p str;*)has_endedp&&has_failedp&&lookaheadp=[Some'd']&&linep=3&&columnp=0let%test_=letopenIndent_parserinletstr="a\nb\n c\n d"inletp=run(return()|.white_space|.absolute(return()|.absolute(letter_ws)|.absolute(letter_ws)|.indentedtrue(absolute(return()|.absolute(letter_ws)|.absolute(letter_ws))))|.expect_end)strin(*print p str;*)has_endedp&&has_failedp&&lookaheadp=[Some'd']&&linep=3&&columnp=2let%test_=letopenIndent_parserinletstr="a\n x\n y\nb\nc"inletp=run(return()|.skip_one_or_more_aligned(letter_ws|.indentedtrue(skip_zero_or_more_alignedletter_ws))|.expect_end)strin(*print p str;*)has_endedp&&lookaheadp=[]&&linep=4&&columnp=1