1040            # and values that are `AggregateScore` objects. Each `AggregateScore` object is a
1041            # named tuple with a low, mid, and high value. Each value is a `Score` object, which
1042            # is also a named tuple, that contains the precision, recall, and fmeasure values.
1043            # For more info see the source code: https://github.com/google-research/google-research/blob/master/rouge/scoring.py  # noqa: E5011044            rouge_result = aggregator.aggregate()
1045
1046            for metric, value in rouge_result.items():

1039            # The aggregator returns a dictionary with keys coresponding to the rouge metric
1040            # and values that are `AggregateScore` objects. Each `AggregateScore` object is a
1041            # named tuple with a low, mid, and high value. Each value is a `Score` object, which
1042            # is also a named tuple, that contains the precision, recall, and fmeasure values.1043            # For more info see the source code: https://github.com/google-research/google-research/blob/master/rouge/scoring.py  # noqa: E501
1044            rouge_result = aggregator.aggregate()
1045

1038                aggregator.add_scores(score)
1039            # The aggregator returns a dictionary with keys coresponding to the rouge metric
1040            # and values that are `AggregateScore` objects. Each `AggregateScore` object is a
1041            # named tuple with a low, mid, and high value. Each value is a `Score` object, which1042            # is also a named tuple, that contains the precision, recall, and fmeasure values.
1043            # For more info see the source code: https://github.com/google-research/google-research/blob/master/rouge/scoring.py  # noqa: E501
1044            rouge_result = aggregator.aggregate()

1037            for score in rouge_scores_list:
1038                aggregator.add_scores(score)
1039            # The aggregator returns a dictionary with keys coresponding to the rouge metric
1040            # and values that are `AggregateScore` objects. Each `AggregateScore` object is a1041            # named tuple with a low, mid, and high value. Each value is a `Score` object, which
1042            # is also a named tuple, that contains the precision, recall, and fmeasure values.
1043            # For more info see the source code: https://github.com/google-research/google-research/blob/master/rouge/scoring.py  # noqa: E501

1036            ]
1037            for score in rouge_scores_list:
1038                aggregator.add_scores(score)
1039            # The aggregator returns a dictionary with keys coresponding to the rouge metric1040            # and values that are `AggregateScore` objects. Each `AggregateScore` object is a
1041            # named tuple with a low, mid, and high value. Each value is a `Score` object, which
1042            # is also a named tuple, that contains the precision, recall, and fmeasure values.

1027            # `test_step()` function. For each batch a list containing the rouge scores
1028            # for each example exists under the key "rouge_scores" in `batch_list`. Thus,
1029            # the below list comprehension loops through the list of outputs and grabs the
1030            # items stored under the "rouge_scores" key. Then it flattens the list of lists1031            # to a list of rouge score objects that can be added to the `aggregator`.
1032            rouge_scores_list = [
1033                rouge_score_set

1026            # In `outputs` there is an entry for each batch that was passwed through the
1027            # `test_step()` function. For each batch a list containing the rouge scores
1028            # for each example exists under the key "rouge_scores" in `batch_list`. Thus,
1029            # the below list comprehension loops through the list of outputs and grabs the1030            # items stored under the "rouge_scores" key. Then it flattens the list of lists
1031            # to a list of rouge score objects that can be added to the `aggregator`.
1032            rouge_scores_list = [

1025
1026            # In `outputs` there is an entry for each batch that was passwed through the
1027            # `test_step()` function. For each batch a list containing the rouge scores
1028            # for each example exists under the key "rouge_scores" in `batch_list`. Thus,1029            # the below list comprehension loops through the list of outputs and grabs the
1030            # items stored under the "rouge_scores" key. Then it flattens the list of lists
1031            # to a list of rouge score objects that can be added to the `aggregator`.

1007    def test_epoch_end(self, outputs):
1008        """
1009        Called at the end of a testing epoch: `PyTorch Lightning Documentation <https://pytorch-lightning.readthedocs.io/en/latest/api/pytorch_lightning.core.html#pytorch_lightning.core.LightningModule.test_epoch_end>`__
1010        Finds the mean of all the metrics logged by :meth:`~extractive.ExtractiveSummarizer.test_step`.1011        """  # noqa: E501
1012        # Get the accuracy metrics over all testing runs
1013        avg_test_acc = torch.stack([x["test_acc"] for x in outputs]).mean()

1006
1007    def test_epoch_end(self, outputs):
1008        """
1009        Called at the end of a testing epoch: `PyTorch Lightning Documentation <https://pytorch-lightning.readthedocs.io/en/latest/api/pytorch_lightning.core.html#pytorch_lightning.core.LightningModule.test_epoch_end>`__1010        Finds the mean of all the metrics logged by :meth:`~extractive.ExtractiveSummarizer.test_step`.
1011        """  # noqa: E501
1012        # Get the accuracy metrics over all testing runs

 977                current_prediction = "<q>".join(current_prediction)
 978                predictions.append(current_prediction)
 979            else:
 980                # Convert `current_prediction` from list to string with a newline between each 981                # item/sentence. `rouge-score` splits sentences by newline.
 982                current_prediction = "\n".join(current_prediction)
 983                target = target.replace("<q>", "\n")

 972                # item/sentence. In ROUGE 1.5.5 (`pyrouge`), a "\n" indicates sentence
 973                # boundaries but the below "save_gold.txt" and "save_pred.txt" could not be
 974                # created if each sentence had to be separated by a newline. Thus, each
 975                # sentence is separated by a "<q>" token and is then converted to a newline 976                # in `helpers.test_rouge`.
 977                current_prediction = "<q>".join(current_prediction)
 978                predictions.append(current_prediction)

 970            if self.hparams.test_use_pyrouge:
 971                # Convert `current_prediction` from list to string with a "<q>" between each
 972                # item/sentence. In ROUGE 1.5.5 (`pyrouge`), a "\n" indicates sentence
 973                # boundaries but the below "save_gold.txt" and "save_pred.txt" could not be 974                # created if each sentence had to be separated by a newline. Thus, each
 975                # sentence is separated by a "<q>" token and is then converted to a newline
 976                # in `helpers.test_rouge`.

 968            # for info about the differences between `pyrouge` and `rouge-score`.
 969            # Archive Link: https://web.archive.org/web/20200622205503/https://github.com/google-research/google-research/issues/168  # noqa: E501
 970            if self.hparams.test_use_pyrouge:
 971                # Convert `current_prediction` from list to string with a "<q>" between each 972                # item/sentence. In ROUGE 1.5.5 (`pyrouge`), a "\n" indicates sentence
 973                # boundaries but the below "save_gold.txt" and "save_pred.txt" could not be
 974                # created if each sentence had to be separated by a newline. Thus, each

 966
 967            # See this issue https://github.com/google-research/google-research/issues/168
 968            # for info about the differences between `pyrouge` and `rouge-score`.
 969            # Archive Link: https://web.archive.org/web/20200622205503/https://github.com/google-research/google-research/issues/168  # noqa: E501 970            if self.hparams.test_use_pyrouge:
 971                # Convert `current_prediction` from list to string with a "<q>" between each
 972                # item/sentence. In ROUGE 1.5.5 (`pyrouge`), a "\n" indicates sentence

 964                ):
 965                    break
 966
 967            # See this issue https://github.com/google-research/google-research/issues/168 968            # for info about the differences between `pyrouge` and `rouge-score`.
 969            # Archive Link: https://web.archive.org/web/20200622205503/https://github.com/google-research/google-research/issues/168  # noqa: E501
 970            if self.hparams.test_use_pyrouge:

 958
 959                # If the testing method is "top_k" and correct number of sentences have been
 960                # added then break the loop and stop adding sentences. If the testing method
 961                # is "greater_k" then we will continue to add all the sentences from `selected_ids` 962                if (self.hparams.test_id_method == "top_k") and (
 963                    len(current_prediction) >= self.hparams.test_k
 964                ):

 957                    current_prediction.append(candidate)
 958
 959                # If the testing method is "top_k" and correct number of sentences have been
 960                # added then break the loop and stop adding sentences. If the testing method 961                # is "greater_k" then we will continue to add all the sentences from `selected_ids`
 962                if (self.hparams.test_id_method == "top_k") and (
 963                    len(current_prediction) >= self.hparams.test_k

 956                ):
 957                    current_prediction.append(candidate)
 958
 959                # If the testing method is "top_k" and correct number of sentences have been 960                # added then break the loop and stop adding sentences. If the testing method
 961                # is "greater_k" then we will continue to add all the sentences from `selected_ids`
 962                if (self.hparams.test_id_method == "top_k") and (

 949                # If trigram blocking is enabled and searching for matching trigrams finds no
 950                # matches then add the candidate to the current prediction list.
 951                # During the predicting process, Trigram Blocking is used to reduce redundancy.
 952                # Given selected summary S and a candidate sentence c, we will skip c is there 953                # exists a trigram overlapping between c and S.
 954                if (not self.hparams.no_test_block_trigrams) and (
 955                    not block_trigrams(candidate, current_prediction)

 948                candidate = source[i].strip()
 949                # If trigram blocking is enabled and searching for matching trigrams finds no
 950                # matches then add the candidate to the current prediction list.
 951                # During the predicting process, Trigram Blocking is used to reduce redundancy. 952                # Given selected summary S and a candidate sentence c, we will skip c is there
 953                # exists a trigram overlapping between c and S.
 954                if (not self.hparams.no_test_block_trigrams) and (

 946                    continue
 947
 948                candidate = source[i].strip()
 949                # If trigram blocking is enabled and searching for matching trigrams finds no 950                # matches then add the candidate to the current prediction list.
 951                # During the predicting process, Trigram Blocking is used to reduce redundancy.
 952                # Given selected summary S and a candidate sentence c, we will skip c is there

 903                    # For the first time the above loop runs, `previous_index` is -1 because no
 904                    # no index has been checked yet. The -1 is necessary to check if the 0th
 905                    # index is skipped. But, if the 0th index is skipped then the values need to be
 906                    # added to the 0th index, not the -1st, so 1 is added to `previous_index` to 907                    # make it 0.
 908                    if previous_index == -1:
 909                        previous_index += 1

 902                if (index not in (previous_index, previous_index + 1)) or value == -1:
 903                    # For the first time the above loop runs, `previous_index` is -1 because no
 904                    # no index has been checked yet. The -1 is necessary to check if the 0th
 905                    # index is skipped. But, if the 0th index is skipped then the values need to be 906                    # added to the 0th index, not the -1st, so 1 is added to `previous_index` to
 907                    # make it 0.
 908                    if previous_index == -1:

 901                # index was skipped because no elements greater than k
 902                if (index not in (previous_index, previous_index + 1)) or value == -1:
 903                    # For the first time the above loop runs, `previous_index` is -1 because no
 904                    # no index has been checked yet. The -1 is necessary to check if the 0th 905                    # index is skipped. But, if the 0th index is skipped then the values need to be
 906                    # added to the 0th index, not the -1st, so 1 is added to `previous_index` to
 907                    # make it 0.