3 dbG@sdZddlmZddlmZmZmZmZmZm Z m Z m Z m Z m Z mZmZmZddlmZmZmZmZmZmZmZmZGdddeZGdddeeZGd d d eZGd d d eZGd ddeZGdddeZGddde Z GdddeZ!GdddeZ"GdddeZ#GdddeZ$eeeeegZ%eeeegZ&ee eegZ'ee eegZ(ee e gZ)GdddeZ*Gddde*Z+Gdd d e*Z,Gd!d"d"e*Z-Gd#d$d$e*Z.Gd%d&d&e*Z/eee"e#e$gZ0eeee!gZ1eeee!gZ2eeee!gZ3Gd'd(d(e*eZ4Gd)d*d*e4Z5Gd+d,d,e4Z6Gd-d.d.e4Z7Gd/d0d0e4Z8d:d6d7Z9e:d8kre9d9S);a Data classes and parser implementations for *incremental* chart parsers, which use dynamic programming to efficiently parse a text. A "chart parser" derives parse trees for a text by iteratively adding "edges" to a "chart". Each "edge" represents a hypothesis about the tree structure for a subsequence of the text. The "chart" is a "blackboard" for composing and combining these hypotheses. A parser is "incremental", if it guarantees that for all i, j where i < j, all edges ending at i are built before any edges ending at j. This is appealing for, say, speech recognizer hypothesis filtering. The main parser class is ``EarleyChartParser``, which is a top-down algorithm, originally formulated by Jay Earley (1970). ) perf_counter) BottomUpPredictCombineRuleBottomUpPredictRuleCachedTopDownPredictRuleChart ChartParserEdgeIEmptyPredictRule"FilteredBottomUpPredictCombineRule!FilteredSingleEdgeFundamentalRuleLeafEdge LeafInitRuleSingleEdgeFundamentalRuleTopDownInitRule)!FeatureBottomUpPredictCombineRuleFeatureBottomUpPredictRule FeatureChartFeatureChartParserFeatureEmptyPredictRule FeatureSingleEdgeFundamentalRuleFeatureTopDownInitRuleFeatureTopDownPredictRulec@sLeZdZddZddZddZddZd d Zd d Zd dZ ddZ dS)IncrementalChartcCs(tdd|jD|_i|_i|_dS)Ncss|] }gVqdS)N).0xrr6/tmp/pip-build-v9q4h5k9/nltk/nltk/parse/earleychart.py @sz.IncrementalChart.initialize..)tuple _positions _edgelistsZ _edge_to_cpls_indexes)selfrrr initialize>szIncrementalChart.initializecCs t|jS)N)list iteredges)r"rrredgesIszIncrementalChart.edgescCsdd|jDS)Ncss|]}|D] }|Vq qdS)Nr)redgelistedgerrrrMsz-IncrementalChart.iteredges..)r )r"rrrr%LszIncrementalChart.iteredgesc sr|j|}ikrt|Stj}t|}||jkrB|j|tfdd|D}t|j||j|gS)Nc3s|]}|VqdS)Nr)rkey) restrictionsrrr^sz*IncrementalChart.select..)r itersortedkeysrr! _add_indexget)r"endr*r' restr_keysvalsr)r*rselectOs    zIncrementalChart.selectcsx"|D]}tt|std|qWtdd|jD}|j|<xTt|jD]F\}}||}x4|D],tfdd|D}|j|gj qfWqPWdS)NzBad restriction: %scss|] }iVqdS)Nr)rrrrrrhsz.IncrementalChart._add_index..c3s|]}t|VqdS)N)getattr)rr))r(rrrns) hasattrr ValueErrorrrr! enumerater setdefaultappend)r"r1r)indexr0r' this_indexr2r)r(rr.as    zIncrementalChart._add_indexcsPj}xB|jjD]4\}}tfdd|D}||j|gjqWdS)Nc3s|]}t|VqdS)N)r4)rr))r(rrrtsz:IncrementalChart._register_with_indexes..)r0r!itemsrr8r9)r"r(r0r1r:r2r)r(r_register_with_indexesqsz'IncrementalChart._register_with_indexescCs|j|jj|dS)N)r r0r9)r"r(rrr _append_edgewszIncrementalChart._append_edgecCst|jdS)N)range num_leaves)r"rrrrzszIncrementalChart._positionsN) __name__ __module__ __qualname__r#r&r%r3r.r=r>rrrrrr=s rc@s$eZdZddZddZddZdS)FeatureIncrementalChartc stj|}ikrt|Stj}t|}|jkrBj|tfdd|D}tj||j|gS)Nc3s|]}j|VqdS)N)_get_type_if_possible)rr))r*r"rrrsz1FeatureIncrementalChart.select..)r r+r,r-rr!r.r/)r"r0r*r'r1r2r)r*r"rr3s    zFeatureIncrementalChart.selectcsx"|D]}tt|std|qWtddjD}j|<xVtjD]H\}}||}x6|D].tfdd|D}|j|gj qfWqPWdS)NzBad restriction: %scss|] }iVqdS)Nr)rrrrrrsz5FeatureIncrementalChart._add_index..c3s |]}jt|VqdS)N)rFr4)rr))r(r"rrrs) r5rr6rrr!r7r r8r9)r"r1r)r:r0r'r;r2r)r(r"rr.s      z"FeatureIncrementalChart._add_indexcsRj}xDjjD]6\}}tfdd|D}||j|gjqWdS)Nc3s |]}jt|VqdS)N)rFr4)rr))r(r"rrrszAFeatureIncrementalChart._register_with_indexes..)r0r!r<rr8r9)r"r(r0r1r:r2r)r(r"rr=s z.FeatureIncrementalChart._register_with_indexesN)rBrCrDr3r.r=rrrrrE~srEc@seZdZddZdS)CompleteFundamentalRuleccsN|j}x@|j||d|jdD]&}|j|j}|j|||r |Vq WdS)NT)startr0 is_completelhs)r0r3nextsymZmove_dot_forwardZinsert_with_backpointer)r"chartgrammar left_edger0 right_edgenew_edgerrr_apply_incompletes z)CompleteFundamentalRule._apply_incompleteN)rBrCrDrQrrrrrGsrGc@seZdZeZddZdS) CompleterRuleccs$t|ts |jj|||EdHdS)N) isinstancer _fundamental_ruleapply)r"rLrMr(rrrrUs zCompleterRule.applyN)rBrCrDrGrTrUrrrrrRsrRc@seZdZeZddZdS) ScannerRuleccs$t|tr |jj|||EdHdS)N)rSr rTrU)r"rLrMr(rrrrUs zScannerRule.applyN)rBrCrDrGrTrUrrrrrVsrVc@s eZdZdS) PredictorRuleN)rBrCrDrrrrrWsrWc@seZdZddZdS)FilteredCompleteFundamentalRuleccs |jr|j|||EdHdS)N)rIZ_apply_complete)r"rLrMr(rrrrUsz%FilteredCompleteFundamentalRule.applyN)rBrCrDrUrrrrrXsrXc@seZdZddZdS)FeatureCompleteFundamentalRuleccsH|j}|j}x4|j||d|jdD]}|j||||EdHq&WdS)NT)rHr0rIrJ)rTr0r3rKrU)r"rLrMrNfrr0rOrrrrQs z0FeatureCompleteFundamentalRule._apply_incompleteN)rBrCrDrQrrrrrYsrYc@seZdZeZdS)FeatureCompleterRuleN)rBrCrDrYrTrrrrr[sr[c@seZdZeZdS)FeatureScannerRuleN)rBrCrDrYrTrrrrr\sr\c@s eZdZdS)FeaturePredictorRuleN)rBrCrDrrrrr]sr]c@s,eZdZdZeddefddZd ddZdS) IncrementalChartParsera An *incremental* chart parser implementing Jay Earley's parsing algorithm: | For each index end in [0, 1, ..., N]: | For each edge such that edge.end = end: | If edge is incomplete and edge.next is not a part of speech: | Apply PredictorRule to edge | If edge is incomplete and edge.next is a part of speech: | Apply ScannerRule to edge | If edge is complete: | Apply CompleterRule to edge | Return any complete parses in the chart r2cCsn||_||_||_||_g|_g|_xD|D]<}|jdkrF|jj|q*|jdkr^|jj|q*tdq*WdS)a Create a new Earley chart parser, that uses ``grammar`` to parse texts. :type grammar: CFG :param grammar: The grammar used to parse texts. :type trace: int :param trace: The level of tracing that should be used when parsing a text. ``0`` will generate no tracing output; and higher numbers will produce more verbose tracing output. :type trace_chart_width: int :param trace_chart_width: The default total width reserved for the chart in trace output. The remainder of each line will be used to display edges. :param chart_class: The class that should be used to create the charts used by this parser. rr?z9Incremental inference rules must have NUM_EDGES == 0 or 1N) _grammar_trace_trace_chart_width _chart_class_axioms_inference_rulesZ NUM_EDGESr9r6)r"rMstrategytracetrace_chart_width chart_classrulerrr__init__/s   zIncrementalChartParser.__init__NcCs@|dkr|j}|j}t|}|jj||j|}|j}|j|jd}|r\t|j |x.|j D]$}t|j ||}||||||qdW|j } xt |jdD]} |dkrtd| dt|j| d} xj| r6| j} xV| D]N} t| j ||| }||| |||x&|D]}|j| kr| j|qWqWqWqW|S)Nr?z * Processing queue: )r0)raZ_trace_new_edgesr$r`Zcheck_coveragercrbrAprintZpretty_format_leavesrdrUrer@r3popr0r9)r"tokensrgZtrace_new_edgesrLrMZtrace_edge_widthZaxiomZ new_edgesZinference_rulesr0Zagendar(rjrPrrr chart_parseZs6      z"IncrementalChartParser.chart_parse)N)rBrCrD__doc__BU_LC_INCREMENTAL_STRATEGYrrkrprrrrr^s  %r^c@seZdZddZdS)EarleyChartParsercKstj||tf|dS)N)r^rkEARLEY_STRATEGY)r"rM parser_argsrrrrkszEarleyChartParser.__init__N)rBrCrDrkrrrrrs~srsc@seZdZddZdS)IncrementalTopDownChartParsercKstj||tf|dS)N)r^rkTD_INCREMENTAL_STRATEGY)r"rMrurrrrksz&IncrementalTopDownChartParser.__init__N)rBrCrDrkrrrrrvsrvc@seZdZddZdS)IncrementalBottomUpChartParsercKstj||tf|dS)N)r^rkBU_INCREMENTAL_STRATEGY)r"rMrurrrrksz'IncrementalBottomUpChartParser.__init__N)rBrCrDrkrrrrrxsrxc@seZdZddZdS)(IncrementalBottomUpLeftCornerChartParsercKstj||tf|dS)N)r^rkrr)r"rMrurrrrksz1IncrementalBottomUpLeftCornerChartParser.__init__N)rBrCrDrkrrrrrzsrzc@seZdZddZdS) IncrementalLeftCornerChartParsercKs&|jstdtj||tf|dS)NzNIncrementalLeftCornerParser only works for grammars without empty productions.)Z is_nonemptyr6r^rkLC_INCREMENTAL_STRATEGY)r"rMrurrrrks z)IncrementalLeftCornerChartParser.__init__N)rBrCrDrkrrrrr{sr{c@seZdZedefddZdS)FeatureIncrementalChartParsercKs tj||f|||d|dS)N)rfrhri)r^rk)r"rMrfrhrirurrrrksz&FeatureIncrementalChartParser.__init__N)rBrCrD"BU_LC_INCREMENTAL_FEATURE_STRATEGYrErkrrrrr}sr}c@seZdZddZdS)FeatureEarleyChartParsercKstj||tf|dS)N)r}rkEARLEY_FEATURE_STRATEGY)r"rMrurrrrksz!FeatureEarleyChartParser.__init__N)rBrCrDrkrrrrrsrc@seZdZddZdS)$FeatureIncrementalTopDownChartParsercKstj||tf|dS)N)r}rkTD_INCREMENTAL_FEATURE_STRATEGY)r"rMrurrrrksz-FeatureIncrementalTopDownChartParser.__init__N)rBrCrDrkrrrrrsrc@seZdZddZdS)%FeatureIncrementalBottomUpChartParsercKstj||tf|dS)N)r}rkBU_INCREMENTAL_FEATURE_STRATEGY)r"rMrurrrrksz.FeatureIncrementalBottomUpChartParser.__init__N)rBrCrDrkrrrrrsrc@seZdZddZdS)/FeatureIncrementalBottomUpLeftCornerChartParsercKstj||tf|dS)N)r}rkr)r"rMrurrrrksz8FeatureIncrementalBottomUpLeftCornerChartParser.__init__N)rBrCrDrkrrrrrsrTF$I saw John with a dog with my cookiecCsddl}ddl}ddlm}|} |r6tdt| tdt||j} t| tt| |d} t} | j| } t | j | j }t| } |rt ||kst d|rx$|D] }t|qWntdt ||rtd | dS) z0 A demonstration of the Earley parsers. rN) demo_grammarz * Grammarz * Sentence:)rgzNot all parses foundz Nr trees:zTime:)systimenltk.parse.chartrrmsplitrsrrpr$parsesrHlenAssertionError)Z print_timesZ print_grammarZ print_treesrgsentZ numparsesrrrrMroZearleytrLrtreerrrdemos2      r__main__N)TFTrrr);rqrrrrrrrrrr r r r r rrZnltk.parse.featurechartrrrrrrrrrrErGrRrVrWrXrYr[r\r]rtrwryrrr|r^rsrvrxrzr{rrrrr}rrrrrrBrrrrs <(A6    _  *