123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609includeSmart_git_intfmoduleVerbose=structtype'afiber='aLwt.tletsucc()=Lwt.return_unitletprint()=Lwt.return_unitendletgit_capabilities=Mimic.make~name:"git-capabilities"letgit_scheme=Mimic.make~name:"git-scheme"letgit_path=Mimic.make~name:"git-path"letgit_host=Mimic.make~name:"git-host"letgit_ssh_user=Mimic.make~name:"git-ssh-user"letgit_port=Mimic.make~name:"git-port"moduleEndpoint=structtypet={scheme:[`SSHofstring|`Git|`HTTPof(string*string)list|`HTTPSof(string*string)list];port:intoption;path:string;host:[`AddrofIpaddr.t|`Domainof[`host]Domain_name.t];}letppppfedn=letpp_hostppf=function|`Addr(Ipaddr.V4v)->Ipaddr.V4.ppppfv|`Addr(Ipaddr.V6v)->Fmt.pfppf"[IPv6:%a]"Ipaddr.V6.ppv|`Domainv->Domain_name.ppppfvinletpp_portppf=function|Someport->Fmt.pfppf":%d"port|None->()inmatchednwith|{scheme=`SSHuser;path;host;_}->Fmt.pfppf"%s@%a:%s"userpp_hosthostpath|{scheme=`Git;port;path;host}->Fmt.pfppf"git://%a%a/%s"pp_hosthostpp_portportpath|{scheme=`HTTP_;path;port;host}->Fmt.pfppf"http://%a%a/%s"pp_hosthostpp_portportpath|{scheme=`HTTPS_;path;port;host}->Fmt.pfppf"https://%a%a/%s"pp_hosthostpp_portportpathlet(<||>)ab=matchawith|Ok_->a|Error_->(matchb()withOk_asr->r|Error_->a)letof_stringstr=letopenRresultinletparse_sshx=letmax=String.lengthxinEmile.of_string_raw~off:0~len:maxx|>R.reword_error(R.msgf"%a"Emile.pp_error)>>=fun(consumed,m)->matchAstring.String.cut~sep:":"(String.subxconsumed(max-consumed))with|Some("",path)->letuser=String.concat"."(List.map(function`Atomx->x|`Stringx->Fmt.str"%S"x)m.Emile.local)in(matchfstm.Emile.domainwith|`Domainvs->(match(Domain_name.(of_stringsvs>>=host),Ipaddr.V4.of_string(String.concat"."vs))with|_,Okipv4->R.ok(`Addr(Ipaddr.V4ipv4))|Okv,_->R.ok(`Domainv)|(Error_aserr),_->err)|`Literalv->Domain_name.of_stringv>>=Domain_name.host>>|funv->`Domainv|`Addr(Emile.IPv4v)->R.ok(`Addr(Ipaddr.V4v))|`Addr(Emile.IPv6v)->R.ok(`Addr(Ipaddr.V6v))|v->R.error_msgf"Invalid hostname: %a"Emile.pp_domainv)>>=funhost->R.ok{scheme=`SSHuser;path;port=None;host}|_->R.error_msg"invalid pattern"inletparse_urix=leturi=Uri.of_stringxinletpath=Uri.pathuriinlethoststr=(Domain_name.of_stringstr>>=Domain_name.host>>|funx->`Domainx)<||>fun()->Ipaddr.of_stringstr>>|funx->`AddrxinmatchUri.schemeuri,Uri.hosturi,Uri.porturiwith|Some"git",Somestr,port->hoststr>>=funhost->R.ok{scheme=`Git;path;port;host}|Some"http",Somestr,port->hoststr>>=funhost->R.ok{scheme=`HTTP[];path;port;host}|Some"https",Somestr,port->hoststr>>=funhost->R.ok{scheme=`HTTPS[];path;port;host}|_->R.error_msgf"invalid uri: %a"Uri.ppuriinparse_sshstr<||>(fun()->parse_uristr)|>R.reword_error(fun_->R.msgf"Invalid endpoint: %s"str)letwith_headers_if_httpheaders({scheme;_}asedn)=matchschemewith|`SSH_|`Git->edn|`HTTP_->{ednwithscheme=`HTTPheaders}|`HTTPS_->{ednwithscheme=`HTTPSheaders}letto_ctxednctx=letscheme=matchedn.schemewith|`Git->`Git|`SSH_->`SSH|`HTTP_->`HTTP|`HTTPS_->`HTTPSinletssh_user=matchedn.schemewith`SSHuser->Someuser|_->Noneinctx|>Mimic.addgit_schemescheme|>Mimic.addgit_pathedn.path|>Mimic.addgit_hostedn.host|>functx->Option.fold~none:ctx~some:(funv->Mimic.addgit_ssh_uservctx)ssh_user|>functx->Option.fold~none:ctx~some:(funv->Mimic.addgit_portvctx)edn.portendmoduleMake(Scheduler:Sigs.SCHEDwithtype+'as='aLwt.t)(Pack:APPENDwithtype+'afiber='aLwt.t)(Index:APPENDwithtype+'afiber='aLwt.t)(HTTP:HTTP)(Uid:UID)(Ref:Sigs.REF)=structletsrc=Logs.Src.create"git-fetch"moduleLog=(valLogs.src_logsrc:Logs.LOG)moduleThin=Carton_lwt.Thin.Make(Uid)letfs=letopenRresultinletopenLwt.InfixinThin.{create=(funtpath->Pack.create~mode:Pack.RdWrtpath>|=R.reword_error(R.msgf"%a"Pack.pp_error));append=Pack.append;map=Pack.map;close=(funtfd->Pack.closetfd>|=R.reword_error(R.msgf"%a"Pack.pp_error));}(* XXX(dinosaure): abstract it? *)letdigest:kind:[`A|`B|`C|`D]->?off:int->?len:int->Bigstringaf.t->Uid.t=fun~kind?(off=0)?lenbuf->letlen=matchlenwithSomelen->len|None->Bigstringaf.lengthbuf-offinletctx=Uid.emptyinletfeed_stringctxstr=letoff=0andlen=String.lengthstrinUid.feedctx(Bigstringaf.of_string~off~lenstr)inletctx=matchkindwith|`A->feed_stringctx(Fmt.str"commit %d\000"len)|`B->feed_stringctx(Fmt.str"tree %d\000"len)|`C->feed_stringctx(Fmt.str"blob %d\000"len)|`D->feed_stringctx(Fmt.str"tag %d\000"len)inletctx=Uid.feedctx~off~lenbufinUid.getctxlet(>>?)=Lwt_result.bindmoduleCartonSched=Carton.Make(Lwt)letfinish_itt~pack~weight~whereoffsets=letopenLwt.InfixinPack.create~mode:Pack.Rdtpack>>?funfd->letzl_buffer=De.bigstring_createDe.io_buffer_sizeinletallocatebits=De.make_window~bitsinletpack=Carton.Dec.makefd~allocate~z:zl_buffer~uid_ln:Uid.length~uid_rw:Uid.of_raw_string(funuid->Hashtbl.findwhereuid)inletmapfd~poslen=letmax=Int64.subweightposinletlen=minmax(Int64.of_intlen)inletlen=Int64.to_intleninPack.maptfd~posleninletrecgoentries=function|[]->Lwt.returnentries|(offset,crc)::offsets->Lwt.catch(fun()->letweight=Carton.Dec.weight_of_offset~mappack~weight:Carton.Dec.nulloffsetinletraw=Carton.Dec.make_raw~weightinletv=Carton.Dec.of_offset~mappackraw~cursor:offsetinletkind=Carton.Dec.kindvinletraw=Carton.Dec.rawvinletlen=Carton.Dec.lenvinletuid=digest~kind~off:0~lenrawingo({Carton.Dec.Idx.offset;crc;uid}::entries)offsets)(funexn->Printexc.print_backtracestdout;Lwt.failexn)ingo[]offsets>>=funentries->Pack.closetfd>>?fun()->Lwt.return_okentriesletrun_pck~light_load~heavy_loadstreamt~src~dst=letopenRresultinletopenLwt.InfixinLwt.catch(fun()->Log.debug(funm->m"Start to verify the given stream.");Thin.verify~digest~threads:1tsrcfsstream)(function|Failureerr->Lwt.return_error(R.msgerr)|Invalid_argumenterr->Lwt.return_error(R.msgerr)|exn->Lwt.return_error(`Exnexn))>>=function|Error_aserr->Lwt.returnerr|Ok(_,[],[],entries,_weight,uid)->Log.debug(funm->m"Given PACK file is not thin, move it!");Pack.movet~src~dst>|=R.reword_error(R.msgf"%a"Pack.pp_error)>>?fun()->Lwt.return_ok(uid,Array.of_listentries)|Ok(n,uids,unresolveds,entries,weight,_uid)->Log.debug(funm->m"Given PACK file is thin, canonicalize!");Thin.canonicalize~light_load~heavy_load~src~dsttfsnuidsweight>>?fun(shift,weight,uid,entries')->letwhere=Hashtbl.create0x100inletentries=letfold({Carton.Dec.Idx.offset;uid;_}asentry)=letoffset=Int64.addoffsetshiftinHashtbl.addwhereuidoffset;{entrywithCarton.Dec.Idx.offset}inList.mapfoldentriesinList.iter(fun{Carton.Dec.Idx.offset;uid;_}->Hashtbl.addwhereuidoffset)entries';letunresolveds=letfold(offset,crc)=Int64.addoffsetshift,crcinList.mapfoldunresolvedsinfinish_it~pack:dst~weight~wheretunresolveds>|=R.reword_error(R.msgf"%a"Pack.pp_error)>>?funentries''->letentries=List.rev_appendentries'entriesinletentries=List.rev_appendentries''entriesinLwt.return_ok(uid,Array.of_listentries)moduleEnc=Carton.Dec.Idx.N(Uid)letrun_idxt~dst~packentries=letopenLwt.Infixinletencoder=Enc.encoder`Manual~packentriesinletbuf=Bigstringaf.createDe.io_buffer_sizeinEnc.dstencoderbuf0(Bigstringaf.lengthbuf);Index.create~mode:Index.Wrtdst>>?funfd->letrecgo=function|`Partial->letlen=Bigstringaf.lengthbuf-Enc.dst_remencoderinIndex.appendtfd(Bigstringaf.substringbuf~off:0~len)>>=fun()->Enc.dstencoderbuf0(Bigstringaf.lengthbuf);go(Enc.encodeencoder`Await)|`Ok->Lwt.return_ok()ingo(Enc.encodeencoder`Await)>>?fun()->Index.closetfdletrun~light_load~heavy_loadstreamt_pckt_idx~src~dst~idx=letopenRresultinletopenLwt.Infixinrun_pck~light_load~heavy_loadstreamt_pck~src~dst>>?fun(pack,entries)->run_idxt_idx~dst:idx~packentries>|=R.reword_error(R.msgf"%a"Index.pp_error)>>?fun()->Lwt.return_okpackmoduleFlow=Unixiz.Make(Mimic)moduleFetch=Nss.Fetch.Make(Scheduler)(Lwt)(Flow)(Uid)(Ref)modulePush=Nss.Push.Make(Scheduler)(Lwt)(Flow)(Uid)(Ref)letfetch_v1?(uses_git_transport=false)~push_stdout~push_stderr~capabilitiespath~ctx?deepen?wanthoststoreaccessfetch_cfgpack=letopenLwt.InfixinMimic.resolvectx>>=function|Error_aserr->letpp_hostppf=function|`Domainv->Domain_name.ppppfv|`Addrv->Ipaddr.ppppfvinLog.err(funm->m"%a not found"pp_hosthost);packNone;Lwt.returnerr|Okflow->Lwt.try_bind(fun()->Fetch.fetch_v1~uses_git_transport~push_stdout~push_stderr~capabilities?deepen?want~hostpath(Flow.makeflow)storeaccessfetch_cfg(fun(payload,off,len)->letv=String.subpayloadoffleninpack(Some(v,0,len))))(funrefs->packNone;Mimic.closeflow>>=fun()->Lwt.return_okrefs)(funexn->packNone;Mimic.closeflow>>=fun()->Lwt.failexn)moduleFlow_http=structtype+'afiber='aLwt.ttypet={mutableic:string;mutableoc:string;mutablepos:int;uri:Uri.t;headers:(string*string)list;ctx:Mimic.ctx;}typeerror=[`Msgofstring]letpp_error=Rresult.R.pp_msgletsendtraw=letoc=t.oc^Cstruct.to_stringrawint.oc<-oc;Lwt.return_ok(Cstruct.lenraw)letrecrecvtraw=ift.pos=String.lengtht.icthen(letopenLwt.Infixin(HTTP.post~ctx:t.ctx~headers:t.headerst.urit.oc>|=Rresult.(R.reword_error(R.msgf"%a"HTTP.pp_error)))>>?fun(_resp,contents)->t.ic<-t.ic^contents;recvtraw)elseletlen=min(String.lengtht.ic-t.pos)(Cstruct.lenraw)inCstruct.blit_from_stringt.ict.posraw0len;t.pos<-t.pos+len;Lwt.return_ok(`Inputlen)endmoduleFetch_http=Nss.Fetch.Make(Scheduler)(Lwt)(Flow_http)(Uid)(Ref)lethttp_fetch_v1~push_stdout~push_stderr~capabilities~ctxuri?(headers=[])endpointpath?deepen?wantstoreaccessfetch_cfgpack=letopenRresultinletopenLwt.Infixinleturi0=Fmt.str"%a/info/refs?service=git-upload-pack"Uri.ppuriinleturi0=Uri.of_stringuri0inLog.debug(funm->m"GET %a"Uri.ppuri0);HTTP.get~ctx~headersuri0>|=R.reword_error(R.msgf"%a"HTTP.pp_error)>>?fun(_resp,contents)->leturi1=Fmt.str"%a/git-upload-pack"Uri.ppuriinleturi1=Uri.of_stringuri1inletflow={Flow_http.ic=contents;pos=0;oc="";uri=uri1;headers;ctx}inFetch_http.fetch_v1~push_stdout~push_stderr~capabilities?deepen?want~host:endpointpathflowstoreaccessfetch_cfg(fun(payload,off,len)->letv=String.subpayloadoffleninpack(Some(v,0,len)))>>=funrefs->packNone;Lwt.return_okrefsletdefault_capabilities=[`Side_band_64k;`Multi_ack_detailed;`Ofs_delta;`Thin_pack;`Report_status;]letfetch?(push_stdout=ignore)?(push_stderr=ignore)~ctx(access,light_load,heavy_load)storeedn?(version=`V1)?(capabilities=default_capabilities)?deepenwantt_pckt_idx~src~dst~idx=letopenRresultinletopenLwt.Infixinlethost=edn.Endpoint.hostinletpath=edn.pathinletstream,pusher=Lwt_stream.create()inletpusher_with_logging=function|Some(_,_,len)asv->Log.debug(funm->m"Download %d byte(s) of the PACK file."len);pusherv|None->Log.debug(funm->m"End of pack.");pusherNoneinletstream()=Lwt_stream.getstreaminletctx=Mimic.addgit_capabilities`Rd(Endpoint.to_ctxednctx)in(* XXX(dinosaure): such trick is only about SSH. Indeed, when we use SSH, we
should/must? know if we want to fetch or push. If we want to fetch, we
will call git-upload-pack. To be able to pass this information to the
"connect" function of SSH (whatever the implementation of SSH), we fill
the given [ctx] with [`Rd]. *)letrun=matchversion,edn.schemewith|`V1,((`Git|`SSH_)asscheme)->letfetch_cfg=Nss.Fetch.configurationcapabilitiesinletuses_git_transport=matchschemewith`Git->true|`SSH_->falseinletrun()=Lwt.both(fetch_v1~push_stdout~push_stderr~uses_git_transport~capabilitiespath~ctx?deepen~wanthoststoreaccessfetch_cfgpusher_with_logging)(run~light_load~heavy_loadstreamt_pckt_idx~src~dst~idx)>>=fun(refs,idx)->matchrefs,idxwith|Okrefs,Okuid->Lwt.return_ok(`Pack(uid,refs))|(Error_aserr),_->Lwt.returnerr|Ok[],_->Lwt.return_ok`Empty|Ok_refs,(Error_aserr)->Lwt.returnerrinrun|`V1,((`HTTP_|`HTTPS_)asscheme)->Log.debug(funm->m"Start an HTTP transmission.");letfetch_cfg=Nss.Fetch.configuration~stateless:truecapabilitiesinletpp_hostppf=function|`Domainv->Domain_name.ppppfv|`Addrv->Ipaddr.ppppfvinleturi,headers=matchschemewith|`HTTPheaders->(Uri.of_string(Fmt.str"http://%a%s.git"pp_hosthostpath),headers)|`HTTPSheaders->(Uri.of_string(Fmt.str"https://%a%s.git"pp_hosthostpath),headers)inletrun()=Lwt.both(http_fetch_v1~push_stdout~push_stderr~capabilities~ctxuri~headershostpath?deepen~wantstoreaccessfetch_cfgpusher_with_logging)(run~light_load~heavy_loadstreamt_pckt_idx~src~dst~idx)>>=fun(refs,idx)->matchrefs,idxwith|Okrefs,Okuid->Lwt.return_ok(`Pack(uid,refs))|(Error_aserr),_->Lwt.returnerr|Ok[],_->Lwt.return_ok`Empty|Ok_refs,(Error_aserr)->Lwt.returnerrinrun|_->assertfalseinLwt.catchrun(function|Failureerr->Lwt.return_error(R.msgerr)|exn->Lwt.return_error(`Exnexn))moduleDelta=Carton_lwt.Enc.Delta(Uid)(Verbose)letdeltify~light_load~heavy_load?(threads=4)(uids:Uid.tlist)=letopenLwt.Infixinletfold(uid:Uid.t)=light_loaduid>|=fun(kind,length)->Carton_lwt.Enc.make_entry~kind~lengthuidinLwt_list.map_pfolduids>|=Array.of_list>>=funentries->Delta.delta~threads:(List.initthreads(fun_thread->heavy_load))~weight:10~uid_ln:Uid.lengthentries>>=funtargets->Lwt.return(entries,targets)letheader=Bigstringaf.create12letpack~(heavy_load:Uid.tCarton_lwt.Enc.load)streamtargets=letopenLwt.Infixinletoffsets=Hashtbl.create(Array.lengthtargets)inletfinduid=matchHashtbl.findoffsetsuidwith|v->Lwt.return_somev|exceptionNot_found->Lwt.return_noneinletuid={Carton.Enc.uid_ln=Uid.length;Carton.Enc.uid_rw=Uid.to_raw_string}inletb={Carton.Enc.o=Bigstringaf.createDe.io_buffer_size;Carton.Enc.i=Bigstringaf.createDe.io_buffer_size;Carton.Enc.q=De.Queue.create0x10000;Carton.Enc.w=De.make_window~bits:15;}inletctx=refUid.emptyinletcursor=ref0inCarton.Enc.header_of_pack~length:(Array.lengthtargets)header012;stream(Some(Bigstringaf.to_stringheader));ctx:=Uid.feed!ctxheader~off:0~len:12;cursor:=!cursor+12;letencode_targetstargets=letencode_targetidx=Hashtbl.addoffsets(Carton.Enc.target_uidtargets.(idx))!cursor;Carton_lwt.Enc.encode_target~b~find~load:heavy_load~uidtargets.(idx)~cursor:!cursor>>=fun(len,encoder)->letrecgoencoder=matchCarton.Enc.N.encode~o:b.oencoderwith|`Flush(encoder,len)->letpayload=Bigstringaf.substringb.o~off:0~leninstream(Somepayload);ctx:=Uid.feed!ctxb.o~off:0~len;cursor:=!cursor+len;letencoder=Carton.Enc.N.dstencoderb.o0(Bigstringaf.lengthb.o)ingoencoder|`End->Lwt.return()inletpayload=Bigstringaf.substringb.o~off:0~leninstream(Somepayload);ctx:=Uid.feed!ctxb.o~off:0~len;cursor:=!cursor+len;letencoder=Carton.Enc.N.dstencoderb.o0(Bigstringaf.lengthb.o)ingoencoderinletrecgoidx=ifidx<Array.lengthtargetsthenencode_targetidx>>=fun()->go(succidx)elseLwt.return()ingo0inencode_targetstargets>>=fun()->letuid=Uid.get!ctx|>Uid.to_raw_stringinstream(Someuid);streamNone;Lwt.return_unitletpack~light_load~heavy_loaduids=letopenLwt.Infixinletstream,pusher=Lwt_stream.create()inletfiber()=deltify~light_load~heavy_loaduids>>=fun(_,targets)->pack~heavy_loadpushertargetsinletstream()=Lwt_stream.getstreaminLwt.asyncfiber;streamletpush?uses_git_transport~ctx~capabilitiespathcmdsendpointstoreaccesspush_cfgpack=letopenLwt.Infixin(* XXX(dinosaure): see [fetch]. *)Mimic.resolvectx>>?funflow->Push.push?uses_git_transport~capabilitiescmds~host:endpointpath(Flow.makeflow)storeaccesspush_cfgpack>>=fun()->Mimic.closeflow>>=fun()->Lwt.return_ok()letpush~ctx(access,light_load,heavy_load)storeedn?(version=`V1)?(capabilities=default_capabilities)cmds=letctx=Mimic.addgit_capabilities`Wr(Endpoint.to_ctxednctx)inletopenRresultinmatchversion,edn.Endpoint.schemewith|`V1,((`Git|`SSH_)asscheme)->letuses_git_transport=matchschemewith`Git->true|`SSH_->falseinlethost=edn.hostinletpath=edn.pathinletpush_cfg=Nss.Push.configuration()inletrun()=push~uses_git_transport~ctx~capabilitiespathcmdshoststoreaccesspush_cfg(pack~light_load~heavy_load)inLwt.catchrun(function|Failureerr->Lwt.return_error(R.msgf"%s"err)|exn->Lwt.return_error(`Exnexn))|_->assertfalseend