summaryrefslogtreecommitdiff
path: root/src/buildstream/source.py
blob: 7fc2e9fc0531bd2d02cba4a28ef7847ae4547d86 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
#
#  Copyright (C) 2016 Codethink Limited
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
#  Authors:
#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
"""
Source - Base source class
==========================

.. _core_source_builtins:

Built-in functionality
----------------------

The Source base class provides built in functionality that may be overridden
by individual plugins.

* Directory

  The ``directory`` variable can be set for all sources of a type in project.conf
  or per source within a element.

  This sets the location within the build root that the content of the source
  will be loaded in to. If the location does not exist, it will be created.

.. _core_source_abstract_methods:

Abstract Methods
----------------
For loading and configuration purposes, Sources must implement the
:ref:`Plugin base class abstract methods <core_plugin_abstract_methods>`.

.. attention::

   In order to ensure that all configuration data is processed at
   load time, it is important that all URLs have been processed during
   :func:`Plugin.configure() <buildstream.plugin.Plugin.configure>`.

   Source implementations *must* either call
   :func:`Source.translate_url() <buildstream.source.Source.translate_url>` or
   :func:`Source.mark_download_url() <buildstream.source.Source.mark_download_url>`
   for every URL that has been specified in the configuration during
   :func:`Plugin.configure() <buildstream.plugin.Plugin.configure>`

Sources expose the following abstract methods. Unless explicitly mentioned,
these methods are mandatory to implement.

* :func:`Source.get_consistency() <buildstream.source.Source.get_consistency>`

  Report the sources consistency state.

* :func:`Source.load_ref() <buildstream.source.Source.load_ref>`

  Load the ref from a specific YAML node

* :func:`Source.get_ref() <buildstream.source.Source.get_ref>`

  Fetch the source ref

* :func:`Source.set_ref() <buildstream.source.Source.set_ref>`

  Set a new ref explicitly

* :func:`Source.track() <buildstream.source.Source.track>`

  Automatically derive a new ref from a symbolic tracking branch

* :func:`Source.fetch() <buildstream.source.Source.fetch>`

  Fetch the actual payload for the currently set ref

* :func:`Source.stage() <buildstream.source.Source.stage>`

  Stage the sources for a given ref at a specified location

* :func:`Source.init_workspace() <buildstream.source.Source.init_workspace>`

  Stage sources in a local directory for use as a workspace.

  **Optional**: If left unimplemented, this will default to calling
  :func:`Source.stage() <buildstream.source.Source.stage>`

* :func:`Source.get_source_fetchers() <buildstream.source.Source.get_source_fetchers>`

  Get the objects that are used for fetching.

  **Optional**: This only needs to be implemented for sources that need to
  download from multiple URLs while fetching (e.g. a git repo and its
  submodules). For details on how to define a SourceFetcher, see
  :ref:`SourceFetcher <core_source_fetcher>`.

* :func:`Source.validate_cache() <buildstream.source.Source.validate_cache>`

  Perform any validations which require the sources to be cached.

  **Optional**: This is completely optional and will do nothing if left unimplemented.

Accessing previous sources
--------------------------
*Since: 1.4*

In the general case, all sources are fetched and tracked independently of one
another. In situations where a source needs to access previous source(s) in
order to perform its own track and/or fetch, following attributes can be set to
request access to previous sources:

* :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_TRACK`

  Indicate that access to previous sources is required during track

* :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH`

  Indicate that access to previous sources is required during fetch

The intended use of such plugins is to fetch external dependencies of other
sources, typically using some kind of package manager, such that all the
dependencies of the original source(s) are available at build time.

When implementing such a plugin, implementors should adhere to the following
guidelines:

* Implementations must be able to store the obtained artifacts in a
  subdirectory.

* Implementations must be able to deterministically generate a unique ref, such
  that two refs are different if and only if they produce different outputs.

* Implementations must not introduce host contamination.


.. _core_source_fetcher:

SourceFetcher - Object for fetching individual URLs
===================================================


Abstract Methods
----------------
SourceFetchers expose the following abstract methods. Unless explicitly
mentioned, these methods are mandatory to implement.

* :func:`SourceFetcher.fetch() <buildstream.source.SourceFetcher.fetch>`

  Fetches the URL associated with this SourceFetcher, optionally taking an
  alias override.

Class Reference
---------------
"""

import os
from contextlib import contextmanager
from typing import Iterable, Iterator, Optional, Tuple, TYPE_CHECKING

from . import _yaml, utils
from .node import MappingNode
from .plugin import Plugin
from .types import Consistency, SourceRef
from ._exceptions import BstError, ImplError, ErrorDomain
from ._loader.metasource import MetaSource
from ._projectrefs import ProjectRefStorage
from ._cachekey import generate_key
from .storage import FileBasedDirectory
from .storage.directory import Directory, VirtualDirectoryError

if TYPE_CHECKING:
    from typing import Any, Dict, Set

    # pylint: disable=cyclic-import
    from ._context import Context
    from ._project import Project
    # pylint: enable=cyclic-import


class SourceError(BstError):
    """This exception should be raised by :class:`.Source` implementations
    to report errors to the user.

    Args:
       message: The breif error description to report to the user
       detail: A possibly multiline, more detailed error message
       reason: An optional machine readable reason string, used for test cases
       temporary: An indicator to whether the error may occur if the operation was run again. (*Since: 1.2*)
    """
    def __init__(self,
                 message: str,
                 *,
                 detail: Optional[str] = None,
                 reason: Optional[str] = None,
                 temporary: bool = False):
        super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason, temporary=temporary)


class SourceFetcher():
    """SourceFetcher()

    This interface exists so that a source that downloads from multiple
    places (e.g. a git source with submodules) has a consistent interface for
    fetching and substituting aliases.

    *Since: 1.2*

    .. attention::

       When implementing a SourceFetcher, remember to call
       :func:`Source.mark_download_url() <buildstream.source.Source.mark_download_url>`
       for every URL found in the configuration data at
       :func:`Plugin.configure() <buildstream.plugin.Plugin.configure>` time.
    """
    def __init__(self):
        self.__alias = None

    #############################################################
    #                      Abstract Methods                     #
    #############################################################
    def fetch(self, alias_override: Optional[str] = None, **kwargs) -> None:
        """Fetch remote sources and mirror them locally, ensuring at least
        that the specific reference is cached locally.

        Args:
           alias_override: The alias to use instead of the default one
               defined by the :ref:`aliases <project_source_aliases>` field
               in the project's config.

        Raises:
           :class:`.SourceError`

        Implementors should raise :class:`.SourceError` if the there is some
        network error or if the source reference could not be matched.
        """
        raise ImplError("SourceFetcher '{}' does not implement fetch()".format(type(self)))

    #############################################################
    #                       Public Methods                      #
    #############################################################
    def mark_download_url(self, url: str) -> None:
        """Identifies the URL that this SourceFetcher uses to download

        This must be called during the fetcher's initialization

        Args:
           url: The url used to download.
        """
        self.__alias = _extract_alias(url)

    #############################################################
    #            Private Methods used in BuildStream            #
    #############################################################

    # Returns the alias used by this fetcher
    def _get_alias(self):
        return self.__alias


class Source(Plugin):
    """Source()

    Base Source class.

    All Sources derive from this class, this interface defines how
    the core will be interacting with Sources.
    """
    # The defaults from the project
    __defaults = None     # type: Optional[Dict[str, Any]]

    BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False
    """Whether access to previous sources is required during track

    When set to True:
      * all sources listed before this source in the given element will be
        fetched before this source is tracked
      * Source.track() will be called with an additional keyword argument
        `previous_sources_dir` where previous sources will be staged
      * this source can not be the first source for an element

    *Since: 1.4*
    """

    BST_REQUIRES_PREVIOUS_SOURCES_FETCH = False
    """Whether access to previous sources is required during fetch

    When set to True:
      * all sources listed before this source in the given element will be
        fetched before this source is fetched
      * Source.fetch() will be called with an additional keyword argument
        `previous_sources_dir` where previous sources will be staged
      * this source can not be the first source for an element

    *Since: 1.4*
    """

    BST_REQUIRES_PREVIOUS_SOURCES_STAGE = False
    """Whether access to previous sources is required during cache

    When set to True:
      * All sources listed before current source in the given element will be
        staged with the source when it's cached.
      * This source can not be the first source for an element.

    *Since: 1.4*
    """

    BST_STAGE_VIRTUAL_DIRECTORY = False
    """Whether we can stage this source directly to a virtual directory

    When set to true, virtual directories can be passed to the source to stage
    to.

    *Since: 1.4*
    """

    def __init__(self,
                 context: 'Context',
                 project: 'Project',
                 meta: MetaSource,
                 *,
                 alias_override: Optional[Tuple[str, str]] = None,
                 unique_id: Optional[int] = None):
        provenance = meta.config.get_provenance()
        # Set element_name member before parent init, as needed for debug messaging
        self.__element_name = meta.element_name         # The name of the element owning this source
        super().__init__("{}-{}".format(meta.element_name, meta.element_index),
                         context, project, provenance, "source", unique_id=unique_id)

        self.__element_index = meta.element_index       # The index of the source in the owning element's source list
        self.__element_kind = meta.element_kind         # The kind of the element owning this source
        self.__directory = meta.directory               # Staging relative directory
        self.__consistency = Consistency.INCONSISTENT   # Cached consistency state
        self.__meta_kind = meta.kind                    # The kind of this source, required for unpickling

        self.__key = None                               # Cache key for source

        # The alias_override is only set on a re-instantiated Source
        self.__alias_override = alias_override          # Tuple of alias and its override to use instead
        self.__expected_alias = None                    # The primary alias
        # Set of marked download URLs
        self.__marked_urls = set()                      # type: Set[str]

        # Collect the composited element configuration and
        # ask the element to configure itself.
        self.__init_defaults(project, meta)
        self.__config = self.__extract_config(meta)
        self.__first_pass = meta.first_pass

        # cached values for commonly access values on the source
        self.__mirror_directory = None                  # type: Optional[str]

        self._configure(self.__config)

    COMMON_CONFIG_KEYS = ['kind', 'directory']
    """Common source config keys

    Source config keys that must not be accessed in configure(), and
    should be checked for using node.validate_keys().
    """

    #############################################################
    #                      Abstract Methods                     #
    #############################################################
    def get_consistency(self) -> int:
        """Report whether the source has a resolved reference

        Returns:
           (:class:`.Consistency`): The source consistency
        """
        raise ImplError("Source plugin '{}' does not implement get_consistency()".format(self.get_kind()))

    def load_ref(self, node: MappingNode) -> None:
        """Loads the *ref* for this Source from the specified *node*.

        Args:
           node: The YAML node to load the ref from

        .. note::

           The *ref* for the Source is expected to be read at
           :func:`Plugin.configure() <buildstream.plugin.Plugin.configure>` time,
           this will only be used for loading refs from alternative locations
           than in the `element.bst` file where the given Source object has
           been declared.

        *Since: 1.2*
        """
        raise ImplError("Source plugin '{}' does not implement load_ref()".format(self.get_kind()))

    def get_ref(self) -> SourceRef:
        """Fetch the internal ref, however it is represented

        Returns:
           (simple object): The internal source reference, or ``None``

        .. note::

           The reference is the user provided (or track resolved) value
           the plugin uses to represent a specific input, like a commit
           in a VCS or a tarball's checksum. Usually the reference is a string,
           but the plugin may choose to represent it with a tuple or such.

           Implementations *must* return a ``None`` value in the case that
           the ref was not loaded. E.g. a ``(None, None)`` tuple is not acceptable.
        """
        raise ImplError("Source plugin '{}' does not implement get_ref()".format(self.get_kind()))

    def set_ref(self, ref: SourceRef, node: MappingNode) -> None:
        """Applies the internal ref, however it is represented

        Args:
           ref (simple object): The internal source reference to set, or ``None``
           node: The same dictionary which was previously passed
                to :func:`Plugin.configure() <buildstream.plugin.Plugin.configure>`

        See :func:`Source.get_ref() <buildstream.source.Source.get_ref>`
        for a discussion on the *ref* parameter.

        .. note::

           Implementors must support the special ``None`` value here to
           allow clearing any existing ref.
        """
        raise ImplError("Source plugin '{}' does not implement set_ref()".format(self.get_kind()))

    def track(self, **kwargs) -> SourceRef:
        """Resolve a new ref from the plugin's track option

        Args:
           previous_sources_dir (str): directory where previous sources are staged.
                                       Note that this keyword argument is available only when
                                       :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_TRACK`
                                       is set to True.

        Returns:
           (simple object): A new internal source reference, or None

        If the backend in question supports resolving references from
        a symbolic tracking branch or tag, then this should be implemented
        to perform this task on behalf of :ref:`bst source track <invoking_source_track>`
        commands.

        This usually requires fetching new content from a remote origin
        to see if a new ref has appeared for your branch or tag. If the
        backend store allows one to query for a new ref from a symbolic
        tracking data without downloading then that is desirable.

        See :func:`Source.get_ref() <buildstream.source.Source.get_ref>`
        for a discussion on the *ref* parameter.
        """
        # Allow a non implementation
        return None

    def fetch(self, **kwargs) -> None:
        """Fetch remote sources and mirror them locally, ensuring at least
        that the specific reference is cached locally.

        Args:
           previous_sources_dir (str): directory where previous sources are staged.
                                       Note that this keyword argument is available only when
                                       :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH`
                                       is set to True.

        Raises:
           :class:`.SourceError`

        Implementors should raise :class:`.SourceError` if the there is some
        network error or if the source reference could not be matched.
        """
        raise ImplError("Source plugin '{}' does not implement fetch()".format(self.get_kind()))

    def stage(self, directory: str) -> None:
        """Stage the sources to a directory

        Args:
           directory: Path to stage the source

        Raises:
           :class:`.SourceError`

        Implementors should assume that *directory* already exists
        and stage already cached sources to the passed directory.

        Implementors should raise :class:`.SourceError` when encountering
        some system error.
        """
        raise ImplError("Source plugin '{}' does not implement stage()".format(self.get_kind()))

    def init_workspace(self, directory: str) -> None:
        """Initialises a new workspace

        Args:
           directory: Path of the workspace to init

        Raises:
           :class:`.SourceError`

        Default implementation is to call
        :func:`Source.stage() <buildstream.source.Source.stage>`.

        Implementors overriding this method should assume that *directory*
        already exists.

        Implementors should raise :class:`.SourceError` when encountering
        some system error.
        """
        self.stage(directory)

    def get_source_fetchers(self) -> Iterable[SourceFetcher]:
        """Get the objects that are used for fetching

        If this source doesn't download from multiple URLs,
        returning None and falling back on the default behaviour
        is recommended.

        Returns:
           The Source's SourceFetchers, if any.

        .. note::

           Implementors can implement this as a generator.

           The :func:`SourceFetcher.fetch() <buildstream.source.SourceFetcher.fetch>`
           method will be called on the returned fetchers one by one,
           before consuming the next fetcher in the list.

        *Since: 1.2*
        """
        return []

    def validate_cache(self) -> None:
        """Implement any validations once we know the sources are cached

        This is guaranteed to be called only once for a given session
        once the sources are known to be
        :attr:`Consistency.CACHED <buildstream.types.Consistency.CACHED>`,
        if source tracking is enabled in the session for this source,
        then this will only be called if the sources become cached after
        tracking completes.

        *Since: 1.4*
        """

    #############################################################
    #                       Public Methods                      #
    #############################################################
    def get_mirror_directory(self) -> str:
        """Fetches the directory where this source should store things

        Returns:
           The directory belonging to this source
        """
        if self.__mirror_directory is None:
            # Create the directory if it doesnt exist
            context = self._get_context()
            directory = os.path.join(context.sourcedir, self.get_kind())
            os.makedirs(directory, exist_ok=True)
            self.__mirror_directory = directory

        return self.__mirror_directory

    def translate_url(self, url: str, *, alias_override: Optional[str] = None, primary: bool = True) -> str:
        """Translates the given url which may be specified with an alias
        into a fully qualified url.

        Args:
           url: A URL, which may be using an alias
           alias_override: Optionally, an URI to override the alias with. (*Since: 1.2*)
           primary: Whether this is the primary URL for the source. (*Since: 1.2*)

        Returns:
           The fully qualified URL, with aliases resolved
        .. note::

           This must be called for every URL in the configuration during
           :func:`Plugin.configure() <buildstream.plugin.Plugin.configure>` if
           :func:`Source.mark_download_url() <buildstream.source.Source.mark_download_url>`
           is not called.
        """
        # Ensure that the download URL is also marked
        self.mark_download_url(url, primary=primary)

        # Alias overriding can happen explicitly (by command-line) or
        # implicitly (the Source being constructed with an __alias_override).
        if alias_override or self.__alias_override:
            url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
            if url_alias:
                if alias_override:
                    url = alias_override + url_body
                else:
                    # Implicit alias overrides may only be done for one
                    # specific alias, so that sources that fetch from multiple
                    # URLs and use different aliases default to only overriding
                    # one alias, rather than getting confused.
                    override_alias = self.__alias_override[0]       # type: ignore
                    override_url = self.__alias_override[1]         # type: ignore
                    if url_alias == override_alias:
                        url = override_url + url_body
            return url
        else:
            project = self._get_project()
            return project.translate_url(url, first_pass=self.__first_pass)

    def mark_download_url(self, url: str, *, primary: bool = True) -> None:
        """Identifies the URL that this Source uses to download

        Args:
           url (str): The URL used to download
           primary (bool): Whether this is the primary URL for the source

        .. note::

           This must be called for every URL in the configuration during
           :func:`Plugin.configure() <buildstream.plugin.Plugin.configure>` if
           :func:`Source.translate_url() <buildstream.source.Source.translate_url>`
           is not called.

        *Since: 1.2*
        """
        # Only mark the Source level aliases on the main instance, not in
        # a reinstantiated instance in mirroring.
        if not self.__alias_override:
            if primary:
                expected_alias = _extract_alias(url)

                assert (self.__expected_alias is None or
                        self.__expected_alias == expected_alias), \
                    "Primary URL marked twice with different URLs"

                self.__expected_alias = expected_alias

        # Enforce proper behaviour of plugins by ensuring that all
        # aliased URLs have been marked at Plugin.configure() time.
        #
        if self._get_configuring():
            # Record marked urls while configuring
            #
            self.__marked_urls.add(url)
        else:
            # If an unknown aliased URL is seen after configuring,
            # this is an error.
            #
            # It is still possible that a URL that was not mentioned
            # in the element configuration can be marked, this is
            # the case for git submodules which might be automatically
            # discovered.
            #
            assert (url in self.__marked_urls or not _extract_alias(url)), \
                "URL was not seen at configure time: {}".format(url)

    def get_project_directory(self) -> str:
        """Fetch the project base directory

        This is useful for sources which need to load resources
        stored somewhere inside the project.

        Returns:
           The project base directory
        """
        project = self._get_project()
        return project.directory

    @contextmanager
    def tempdir(self) -> Iterator[str]:
        """Context manager for working in a temporary directory

        Yields:
           A path to a temporary directory

        This should be used by source plugins directly instead of the tempfile
        module. This one will automatically cleanup in case of termination by
        catching the signal before os._exit(). It will also use the 'mirror
        directory' as expected for a source.
        """
        mirrordir = self.get_mirror_directory()
        with utils._tempdir(dir=mirrordir) as tempdir:
            yield tempdir

    #############################################################
    #       Private Abstract Methods used in BuildStream        #
    #############################################################

    # Returns the local path to the source
    #
    # If the source is locally available, this method returns the absolute
    # path. Otherwise, the return value is None.
    #
    # This is an optimization for local sources and optional to implement.
    #
    # Returns:
    #    (str): The local absolute path, or None
    #
    def _get_local_path(self):
        return None

    #############################################################
    #            Private Methods used in BuildStream            #
    #############################################################

    # Wrapper around preflight() method
    #
    def _preflight(self):
        try:
            self.preflight()
        except BstError as e:
            # Prepend provenance to the error
            raise SourceError("{}: {}".format(self, e), reason=e.reason) from e

    # Update cached consistency for a source
    #
    # This must be called whenever the state of a source may have changed.
    #
    def _update_state(self):

        if self.__consistency < Consistency.CACHED:

            # Source consistency interrogations are silent.
            context = self._get_context()
            with context.messenger.silence():
                self.__consistency = self.get_consistency()  # pylint: disable=assignment-from-no-return

                # Give the Source an opportunity to validate the cached
                # sources as soon as the Source becomes Consistency.CACHED.
                if self.__consistency == Consistency.CACHED:
                    self.validate_cache()

    # Return cached consistency
    #
    def _get_consistency(self):
        return self.__consistency

    # Wrapper function around plugin provided fetch method
    #
    # Args:
    #   previous_sources (list): List of Sources listed prior to this source
    #   fetch_original (bool): whether to fetch full source, or use local CAS
    #
    def _fetch(self, previous_sources):

        if self.BST_REQUIRES_PREVIOUS_SOURCES_FETCH:
            self.__ensure_previous_sources(previous_sources)
            with self.__stage_previous_sources(previous_sources) as staging_directory:
                self.__do_fetch(previous_sources_dir=self.__ensure_directory(staging_directory))
        else:
            self.__do_fetch()

    # Wrapper for stage() api which gives the source
    # plugin a fully constructed path considering the
    # 'directory' option
    #
    def _stage(self, directory):
        directory = self.__ensure_directory(directory)

        self.stage(directory)

    # Wrapper for init_workspace()
    def _init_workspace(self, directory):
        if self.BST_STAGE_VIRTUAL_DIRECTORY:
            directory = FileBasedDirectory(external_directory=directory)

        directory = self.__ensure_directory(directory)

        self.init_workspace(directory)

    # _get_unique_key():
    #
    # Wrapper for get_unique_key() api
    #
    # Args:
    #    include_source (bool): Whether to include the delegated source key
    #
    def _get_unique_key(self, include_source):
        key = {}

        key['directory'] = self.__directory
        if include_source:
            key['unique'] = self.get_unique_key()  # pylint: disable=assignment-from-no-return

        return key

    # _project_refs():
    #
    # Gets the appropriate ProjectRefs object for this source,
    # which depends on whether the owning element is a junction
    #
    # Args:
    #    project (Project): The project to check
    #
    def _project_refs(self, project):
        element_kind = self.__element_kind
        if element_kind == 'junction':
            return project.junction_refs
        return project.refs

    # _load_ref():
    #
    # Loads the ref for the said source.
    #
    # Raises:
    #    (SourceError): If the source does not implement load_ref()
    #
    # Returns:
    #    (ref): A redundant ref specified inline for a project.refs using project
    #
    # This is partly a wrapper around `Source.load_ref()`, it will decide
    # where to load the ref from depending on which project the source belongs
    # to and whether that project uses a project.refs file.
    #
    # Note the return value is used to construct a summarized warning in the
    # case that the toplevel project uses project.refs and also lists refs
    # which will be ignored.
    #
    def _load_ref(self):
        context = self._get_context()
        project = self._get_project()
        toplevel = context.get_toplevel_project()
        redundant_ref = None

        element_name = self.__element_name
        element_idx = self.__element_index

        def do_load_ref(node):
            try:
                self.load_ref(ref_node)
            except ImplError as e:
                raise SourceError("{}: Storing refs in project.refs is not supported by '{}' sources"
                                  .format(self, self.get_kind()),
                                  reason="unsupported-load-ref") from e

        # If the main project overrides the ref, use the override
        if project is not toplevel and toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
            refs = self._project_refs(toplevel)
            ref_node = refs.lookup_ref(project.name, element_name, element_idx)
            if ref_node is not None:
                do_load_ref(ref_node)

        # If the project itself uses project.refs, clear the ref which
        # was already loaded via Source.configure(), as this would
        # violate the rule of refs being either in project.refs or in
        # the elements themselves.
        #
        elif project.ref_storage == ProjectRefStorage.PROJECT_REFS:

            # First warn if there is a ref already loaded, and reset it
            redundant_ref = self.get_ref()  # pylint: disable=assignment-from-no-return
            if redundant_ref is not None:
                self.set_ref(None, {})

            # Try to load the ref
            refs = self._project_refs(project)
            ref_node = refs.lookup_ref(project.name, element_name, element_idx)
            if ref_node is not None:
                do_load_ref(ref_node)

        return redundant_ref

    # _set_ref()
    #
    # Persists the ref for this source. This will decide where to save the
    # ref, or refuse to persist it, depending on active ref-storage project
    # settings.
    #
    # Args:
    #    new_ref (smth): The new reference to save
    #    save (bool): Whether to write the new reference to file or not
    #
    # Returns:
    #    (bool): Whether the ref has changed
    #
    # Raises:
    #    (SourceError): In the case we encounter errors saving a file to disk
    #
    def _set_ref(self, new_ref, *, save):

        context = self._get_context()
        project = self._get_project()
        toplevel = context.get_toplevel_project()
        toplevel_refs = self._project_refs(toplevel)
        provenance = self._get_provenance()

        element_name = self.__element_name
        element_idx = self.__element_index

        #
        # Step 1 - Obtain the node
        #
        node = {}
        if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
            node = toplevel_refs.lookup_ref(project.name, element_name, element_idx, write=True)

        if project is toplevel and not node:
            node = provenance._node

        # Ensure the node is not from a junction
        if not toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS and provenance._project is not toplevel:
            if provenance._project is project:
                self.warn("{}: Not persisting new reference in junctioned project".format(self))
            elif provenance._project is None:
                assert provenance._filename == ""
                assert provenance._shortname == ""
                raise SourceError("{}: Error saving source reference to synthetic node."
                                  .format(self))
            else:
                raise SourceError("{}: Cannot track source in a fragment from a junction"
                                  .format(provenance._shortname),
                                  reason="tracking-junction-fragment")

        #
        # Step 2 - Set the ref in memory, and determine changed state
        #
        # TODO: we are working on dictionaries here, would be nicer to just work on the nodes themselves
        clean = node.strip_node_info()
        to_modify = node.strip_node_info()

        current_ref = self.get_ref()  # pylint: disable=assignment-from-no-return

        # Set the ref regardless of whether it changed, the
        # TrackQueue() will want to update a specific node with
        # the ref, regardless of whether the original has changed.
        self.set_ref(new_ref, to_modify)

        if current_ref == new_ref or not save:
            # Note: We do not look for and propagate changes at this point
            # which might result in desync depending if something changes about
            # tracking in the future.  For now, this is quite safe.
            return False

        actions = {}
        for k, v in clean.items():
            if k not in to_modify:
                actions[k] = 'del'
            else:
                if v != to_modify[k]:
                    actions[k] = 'mod'
        for k in to_modify.keys():
            if k not in clean:
                actions[k] = 'add'

        def walk_container(container, path):
            # For each step along path, synthesise if we need to.
            # If we're synthesising missing list entries, we know we're
            # doing this for project.refs so synthesise empty dicts for the
            # intervening entries too
            lpath = [step for step in path]
            lpath.append("")  # We know the last step will be a string key
            for step, next_step in zip(lpath, lpath[1:]):
                if type(step) is str:  # pylint: disable=unidiomatic-typecheck
                    # handle dict container
                    if step not in container:
                        if type(next_step) is str:  # pylint: disable=unidiomatic-typecheck
                            container[step] = {}
                        else:
                            container[step] = []
                    container = container[step]
                else:
                    # handle list container
                    if len(container) <= step:
                        while len(container) <= step:
                            container.append({})
                    container = container[step]
            return container

        def process_value(action, container, path, key, new_value):
            container = walk_container(container, path)
            if action == 'del':
                del container[key]
            elif action == 'mod':
                container[key] = new_value
            elif action == 'add':
                container[key] = new_value
            else:
                assert False, \
                    "BUG: Unknown action: {}".format(action)

        roundtrip_cache = {}
        for key, action in actions.items():
            # Obtain the top level node and its file
            if action == 'add':
                provenance = node.get_provenance()
            else:
                provenance = node.get_node(key).get_provenance()

            toplevel_node = provenance._toplevel

            # Get the path to whatever changed
            if action == 'add':
                path = toplevel_node._find(node)
            else:
                full_path = toplevel_node._find(node.get_node(key))
                # We want the path to the node containing the key, not to the key
                path = full_path[:-1]

            roundtrip_file = roundtrip_cache.get(provenance._filename)
            if not roundtrip_file:
                roundtrip_file = roundtrip_cache[provenance._filename] = _yaml.roundtrip_load(
                    provenance._filename,
                    allow_missing=True
                )

            # Get the value of the round trip file that we need to change
            process_value(action, roundtrip_file, path, key, to_modify.get(key))

        #
        # Step 3 - Apply the change in project data
        #
        for filename, data in roundtrip_cache.items():
            # This is our roundtrip dump from the track
            try:
                _yaml.roundtrip_dump(data, filename)
            except OSError as e:
                raise SourceError("{}: Error saving source reference to '{}': {}"
                                  .format(self, filename, e),
                                  reason="save-ref-error") from e

        return True

    # Wrapper for track()
    #
    # Args:
    #   previous_sources (list): List of Sources listed prior to this source
    #
    def _track(self, previous_sources):
        if self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK:
            self.__ensure_previous_sources(previous_sources)
            with self.__stage_previous_sources(previous_sources) \
                    as staging_directory:
                new_ref = self.__do_track(previous_sources_dir=self.__ensure_directory(staging_directory))
        else:
            new_ref = self.__do_track()

        current_ref = self.get_ref()  # pylint: disable=assignment-from-no-return

        if new_ref is None:
            # No tracking, keep current ref
            new_ref = current_ref

        if current_ref != new_ref:
            self.info("Found new revision: {}".format(new_ref))

            # Save ref in local process for subsequent sources
            self._set_ref(new_ref, save=False)

        return new_ref

    # _requires_previous_sources()
    #
    # If a plugin requires access to previous sources at track or fetch time,
    # then it cannot be the first source of an elemenet.
    #
    # Returns:
    #   (bool): Whether this source requires access to previous sources
    #
    def _requires_previous_sources(self):
        return self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK or self.BST_REQUIRES_PREVIOUS_SOURCES_FETCH

    # Returns the alias if it's defined in the project
    def _get_alias(self):
        alias = self.__expected_alias
        project = self._get_project()
        if project.get_alias_uri(alias, first_pass=self.__first_pass):
            # The alias must already be defined in the project's aliases
            # otherwise http://foo gets treated like it contains an alias
            return alias
        else:
            return None

    def _generate_key(self, previous_sources):
        keys = [self._get_unique_key(True)]

        if self.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
            for previous_source in previous_sources:
                keys.append(previous_source._get_unique_key(True))

        self.__key = generate_key(keys)

        sourcecache = self._get_context().sourcecache
        if self.get_kind() == 'workspace' and not sourcecache.contains(self):
            sourcecache.commit(self, [])

    @property
    def _key(self):
        return self.__key

    # Gives a ref path that points to where sources are kept in the CAS
    def _get_source_name(self):
        # @ is used to prevent conflicts with project names
        return "{}/{}".format(
            self.get_kind(),
            self._key)

    def _get_brief_display_key(self):
        context = self._get_context()
        key = self._key

        length = min(len(key), context.log_key_length)
        return key[:length]

    @property
    def _element_name(self):
        return self.__element_name

    # _get_args_for_child_job_pickling(self)
    #
    # Return data necessary to reconstruct this object in a child job process.
    #
    # Returns:
    #    (PluginContext, str, dict): A tuple of (factory, meta_kind, state),
    #    where `factory` is an object that can use `meta_kind` to create an
    #    instance of the same type as `self`. `state` is what we want
    #    `self.__dict__` to be restored to after instantiation in the child
    #    process.
    #
    def _get_args_for_child_job_pickling(self):
        factory = self._get_project().config.source_factory

        # In case you're wondering, note that it doesn't seem to be necessary
        # to make a copy of `self.__dict__` here, because:
        #
        #   o It seems that the default implementation of `_PyObject_GetState`
        #     in `typeobject.c` currently works this way, in CPython.
        #
        #   o The code sketch of how pickling works also returns `self.__dict__`:
        #     https://docs.python.org/3/library/pickle.html#pickling-class-instances
        #
        return factory, self.__meta_kind, self.__dict__

    #############################################################
    #                   Local Private Methods                   #
    #############################################################

    # __clone_for_uri()
    #
    # Clone the source with an alternative URI setup for the alias
    # which this source uses.
    #
    # This is used for iteration over source mirrors.
    #
    # Args:
    #    uri (str): The alternative URI for this source's alias
    #
    # Returns:
    #    (Source): A new clone of this Source, with the specified URI
    #              as the value of the alias this Source has marked as
    #              primary with either mark_download_url() or
    #              translate_url().
    #
    def __clone_for_uri(self, uri):
        project = self._get_project()
        context = self._get_context()
        alias = self._get_alias()
        source_kind = type(self)

        # Rebuild a MetaSource from the current element
        meta = MetaSource(
            self.__element_name,
            self.__element_index,
            self.__element_kind,
            self.get_kind(),
            self.__config,
            self.__directory,
        )

        meta.first_pass = self.__first_pass

        clone = source_kind(context, project, meta,
                            alias_override=(alias, uri),
                            unique_id=self._unique_id)

        # Do the necessary post instantiation routines here
        #
        clone._preflight()
        clone._load_ref()
        clone._update_state()

        return clone

    # Context manager that stages sources in a cas based or temporary file
    # based directory
    @contextmanager
    def __stage_previous_sources(self, sources):
        with self.tempdir() as tempdir:
            directory = FileBasedDirectory(external_directory=tempdir)

            for src in sources:
                if src.BST_STAGE_VIRTUAL_DIRECTORY:
                    src._stage(directory)
                else:
                    src._stage(tempdir)

            if self.BST_STAGE_VIRTUAL_DIRECTORY:
                yield directory
            else:
                yield tempdir

    # Tries to call fetch for every mirror, stopping once it succeeds
    def __do_fetch(self, **kwargs):
        project = self._get_project()
        context = self._get_context()

        # Silence the STATUS messages which might happen as a result
        # of checking the source fetchers.
        with context.messenger.silence():
            source_fetchers = self.get_source_fetchers()

        # Use the source fetchers if they are provided
        #
        if source_fetchers:

            # Use a contorted loop here, this is to allow us to
            # silence the messages which can result from consuming
            # the items of source_fetchers, if it happens to be a generator.
            #
            source_fetchers = iter(source_fetchers)

            while True:

                with context.messenger.silence():
                    try:
                        fetcher = next(source_fetchers)
                    except StopIteration:
                        # as per PEP479, we are not allowed to let StopIteration
                        # thrown from a context manager.
                        # Catching it here and breaking instead.
                        break

                alias = fetcher._get_alias()
                for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
                    try:
                        fetcher.fetch(uri)
                    # FIXME: Need to consider temporary vs. permanent failures,
                    #        and how this works with retries.
                    except BstError as e:
                        last_error = e
                        continue

                    # No error, we're done with this fetcher
                    break

                else:
                    # No break occurred, raise the last detected error
                    raise last_error

        # Default codepath is to reinstantiate the Source
        #
        else:
            alias = self._get_alias()
            if self.__first_pass:
                mirrors = project.first_pass_config.mirrors
            else:
                mirrors = project.config.mirrors
            if not mirrors or not alias:
                self.fetch(**kwargs)
                return

            for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
                new_source = self.__clone_for_uri(uri)
                try:
                    new_source.fetch(**kwargs)
                # FIXME: Need to consider temporary vs. permanent failures,
                #        and how this works with retries.
                except BstError as e:
                    last_error = e
                    continue

                # No error, we're done here
                return

            # Re raise the last detected error
            raise last_error

    # Tries to call track for every mirror, stopping once it succeeds
    def __do_track(self, **kwargs):
        project = self._get_project()
        alias = self._get_alias()
        if self.__first_pass:
            mirrors = project.first_pass_config.mirrors
        else:
            mirrors = project.config.mirrors
        # If there are no mirrors, or no aliases to replace, there's nothing to do here.
        if not mirrors or not alias:
            return self.track(**kwargs)

        # NOTE: We are assuming here that tracking only requires substituting the
        #       first alias used
        for uri in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass)):
            new_source = self.__clone_for_uri(uri)
            try:
                ref = new_source.track(**kwargs)  # pylint: disable=assignment-from-none
            # FIXME: Need to consider temporary vs. permanent failures,
            #        and how this works with retries.
            except BstError as e:
                last_error = e
                continue
            return ref
        raise last_error

    # Ensures a fully constructed path and returns it
    def __ensure_directory(self, directory):

        if not isinstance(directory, Directory):
            if self.__directory is not None:
                directory = os.path.join(directory, self.__directory.lstrip(os.sep))

            try:
                os.makedirs(directory, exist_ok=True)
            except OSError as e:
                raise SourceError("Failed to create staging directory: {}"
                                  .format(e),
                                  reason="ensure-stage-dir-fail") from e

        else:
            if self.__directory is not None:
                try:
                    directory = directory.descend(
                        *self.__directory.lstrip(os.sep).split(os.sep),
                        create=True)
                except VirtualDirectoryError as e:
                    raise SourceError("Failed to descend into staging directory: {}"
                                      .format(e),
                                      reason="ensure-stage-dir-fail") from e

        return directory

    @classmethod
    def __init_defaults(cls, project, meta):
        if cls.__defaults is None:
            if meta.first_pass:
                sources = project.first_pass_config.source_overrides
            else:
                sources = project.source_overrides
            cls.__defaults = sources.get_mapping(meta.kind, default={})

    # This will resolve the final configuration to be handed
    # off to source.configure()
    #
    @classmethod
    def __extract_config(cls, meta):
        config = cls.__defaults.get_mapping('config', default={})
        config = config.clone()

        meta.config._composite(config)
        config._assert_fully_composited()

        return config

    # Ensures that previous sources have been tracked and fetched.
    #
    def __ensure_previous_sources(self, previous_sources):
        for index, src in enumerate(previous_sources):
            # BuildStream should track sources in the order they appear so
            # previous sources should never be in an inconsistent state
            assert src.get_consistency() != Consistency.INCONSISTENT

            if src.get_consistency() == Consistency.RESOLVED:
                src._fetch(previous_sources[0:index])


def _extract_alias(url):
    parts = url.split(utils._ALIAS_SEPARATOR, 1)
    if len(parts) > 1 and not parts[0].lower() in utils._URI_SCHEMES:
        return parts[0]
    else:
        return ""