This project has retired. For details please refer to its Attic page.
DefaultRepositoryProxyHandler xref
View Javadoc
1   package org.apache.archiva.proxy;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *  http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.archiva.checksum.ChecksumAlgorithm;
23  import org.apache.archiva.checksum.ChecksumUtil;
24  import org.apache.archiva.common.filelock.FileLockManager;
25  import org.apache.archiva.configuration.ArchivaConfiguration;
26  import org.apache.archiva.configuration.ProxyConnectorConfiguration;
27  import org.apache.archiva.configuration.ProxyConnectorRuleConfiguration;
28  import org.apache.archiva.model.ArtifactReference;
29  import org.apache.archiva.model.Keys;
30  import org.apache.archiva.policies.DownloadErrorPolicy;
31  import org.apache.archiva.policies.DownloadPolicy;
32  import org.apache.archiva.policies.Policy;
33  import org.apache.archiva.policies.PolicyConfigurationException;
34  import org.apache.archiva.policies.PolicyOption;
35  import org.apache.archiva.policies.PolicyViolationException;
36  import org.apache.archiva.policies.PostDownloadPolicy;
37  import org.apache.archiva.policies.PreDownloadPolicy;
38  import org.apache.archiva.policies.ProxyDownloadException;
39  import org.apache.archiva.policies.urlcache.UrlFailureCache;
40  import org.apache.archiva.proxy.model.NetworkProxy;
41  import org.apache.archiva.proxy.model.ProxyConnector;
42  import org.apache.archiva.proxy.model.ProxyFetchResult;
43  import org.apache.archiva.proxy.model.RepositoryProxyHandler;
44  import org.apache.archiva.components.taskqueue.TaskQueueException;
45  import org.apache.archiva.repository.ManagedRepository;
46  import org.apache.archiva.repository.RemoteRepository;
47  import org.apache.archiva.repository.RemoteRepositoryContent;
48  import org.apache.archiva.repository.RepositoryType;
49  import org.apache.archiva.repository.metadata.base.MetadataTools;
50  import org.apache.archiva.repository.metadata.RepositoryMetadataException;
51  import org.apache.archiva.repository.storage.FilesystemStorage;
52  import org.apache.archiva.repository.storage.StorageAsset;
53  import org.apache.archiva.repository.storage.StorageUtil;
54  import org.apache.archiva.scheduler.ArchivaTaskScheduler;
55  import org.apache.archiva.scheduler.repository.model.RepositoryTask;
56  import org.apache.commons.collections4.CollectionUtils;
57  import org.apache.commons.io.FilenameUtils;
58  import org.apache.commons.lang3.StringUtils;
59  import org.apache.commons.lang3.SystemUtils;
60  import org.apache.tools.ant.types.selectors.SelectorUtils;
61  import org.slf4j.Logger;
62  import org.slf4j.LoggerFactory;
63  import org.slf4j.MarkerFactory;
64  
65  import javax.annotation.PostConstruct;
66  import javax.inject.Inject;
67  import javax.inject.Named;
68  import java.io.IOException;
69  import java.net.MalformedURLException;
70  import java.nio.file.Files;
71  import java.nio.file.Path;
72  import java.nio.file.StandardCopyOption;
73  import java.util.ArrayList;
74  import java.util.Collections;
75  import java.util.HashMap;
76  import java.util.LinkedHashMap;
77  import java.util.List;
78  import java.util.Map;
79  import java.util.Properties;
80  import java.util.concurrent.ConcurrentHashMap;
81  import java.util.concurrent.ConcurrentMap;
82  
83  public abstract class DefaultRepositoryProxyHandler implements RepositoryProxyHandler {
84  
85      protected Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyHandler.class );
86      @Inject
87      protected UrlFailureCache urlFailureCache;
88  
89      @Inject
90      @Named(value = "metadataTools#default")
91      private MetadataTools metadataTools;
92  
93      private Map<String, PreDownloadPolicy> preDownloadPolicies = new HashMap<>(  );
94      private Map<String, PostDownloadPolicy> postDownloadPolicies = new HashMap<>(  );
95      private Map<String, DownloadErrorPolicy> downloadErrorPolicies = new HashMap<>(  );
96      private ConcurrentMap<String, List<ProxyConnector>> proxyConnectorMap = new ConcurrentHashMap<>();
97  
98      @Inject
99      @Named(value = "archivaTaskScheduler#repository")
100     private ArchivaTaskScheduler<RepositoryTask> scheduler;
101 
102     @Inject
103     private ArchivaConfiguration archivaConfiguration;
104 
105     @Inject
106     @Named(value = "fileLockManager#default")
107     private FileLockManager fileLockManager;
108 
109     private Map<String, NetworkProxy> networkProxyMap = new ConcurrentHashMap<>();
110     private List<ChecksumAlgorithm> checksumAlgorithms;
111 
112     @PostConstruct
113     public void initialize()
114     {
115         checksumAlgorithms = ChecksumUtil.getAlgorithms(archivaConfiguration.getConfiguration().getArchivaRuntimeConfiguration().getChecksumTypes());
116     }
117 
118     private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules(String sourceRepository,
119                                                                           String targetRepository,
120                                                                           List<ProxyConnectorRuleConfiguration> all )
121     {
122         List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = new ArrayList<>();
123 
124         for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all )
125         {
126             for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() )
127             {
128                 if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals(
129                     targetRepository, proxyConnector.getTargetRepoId() ) )
130                 {
131                     proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration );
132                 }
133             }
134         }
135 
136         return proxyConnectorRuleConfigurations;
137     }
138 
139     @Override
140     public StorageAsset fetchFromProxies( ManagedRepository repository, ArtifactReference artifact )
141         throws ProxyDownloadException
142     {
143         StorageAsset localFile = toLocalFile( repository, artifact );
144 
145         Properties requestProperties = new Properties();
146         requestProperties.setProperty( "filetype", "artifact" );
147         requestProperties.setProperty( "version", artifact.getVersion() );
148         requestProperties.setProperty( "managedRepositoryId", repository.getId() );
149 
150         List<ProxyConnector> connectors = getProxyConnectors( repository );
151         Map<String, Exception> previousExceptions = new LinkedHashMap<>();
152         for ( ProxyConnector connector : connectors )
153         {
154             if ( !connector.isEnabled() )
155             {
156                 continue;
157             }
158 
159             RemoteRepository targetRepository = connector.getTargetRepository();
160             requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
161 
162             String targetPath = targetRepository.getContent().toPath( artifact );
163 
164             if ( SystemUtils.IS_OS_WINDOWS )
165             {
166                 // toPath use system PATH_SEPARATOR so on windows url are \ which doesn't work very well :-)
167                 targetPath = FilenameUtils.separatorsToUnix( targetPath );
168             }
169 
170             try
171             {
172                 StorageAsset downloadedFile =
173                     transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
174                                   true );
175 
176                 if ( fileExists(downloadedFile) )
177                 {
178                     log.debug( "Successfully transferred: {}", downloadedFile.getPath() );
179                     return downloadedFile;
180                 }
181             }
182             catch ( NotFoundException e )
183             {
184                 log.debug( "Artifact {} not found on repository \"{}\".", Keys.toKey( artifact ),
185                            targetRepository.getId() );
186             }
187             catch ( NotModifiedException e )
188             {
189                 log.debug( "Artifact {} not updated on repository \"{}\".", Keys.toKey( artifact ),
190                            targetRepository.getId() );
191             }
192             catch ( ProxyException e )
193             {
194                 validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
195                                   targetRepository.getContent(), localFile, e, previousExceptions );
196             }
197         }
198 
199         if ( !previousExceptions.isEmpty() )
200         {
201             throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
202                                               previousExceptions );
203         }
204 
205         log.debug( "Exhausted all target repositories, artifact {} not found.", Keys.toKey( artifact ) );
206 
207         return null;
208     }
209 
210     @Override
211     public StorageAsset fetchFromProxies( ManagedRepository repository, String path )
212     {
213         StorageAsset localFile = repository.getAsset( path );
214 
215         // no update policies for these paths
216         if ( localFile.exists() )
217         {
218             return null;
219         }
220 
221         Properties requestProperties = new Properties();
222         requestProperties.setProperty( "filetype", "resource" );
223         requestProperties.setProperty( "managedRepositoryId", repository.getId() );
224 
225         List<ProxyConnector> connectors = getProxyConnectors( repository );
226         for ( ProxyConnector connector : connectors )
227         {
228             if ( !connector.isEnabled() )
229             {
230                 continue;
231             }
232 
233             RemoteRepository targetRepository = connector.getTargetRepository();
234             requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
235 
236             String targetPath = path;
237 
238             try
239             {
240                 StorageAsset downloadedFile =
241                     transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
242                                   false );
243 
244                 if ( fileExists( downloadedFile ) )
245                 {
246                     log.debug( "Successfully transferred: {}", downloadedFile.getPath() );
247                     return downloadedFile;
248                 }
249             }
250             catch ( NotFoundException e )
251             {
252                 log.debug( "Resource {} not found on repository \"{}\".", path,
253                            targetRepository.getId() );
254             }
255             catch ( NotModifiedException e )
256             {
257                 log.debug( "Resource {} not updated on repository \"{}\".", path,
258                            targetRepository.getId() );
259             }
260             catch ( ProxyException e )
261             {
262                 log.warn(
263                     "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
264                     targetRepository.getId(), path, e.getMessage() );
265                 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
266                            "Transfer error from repository \"{}"
267                                + "\" for resource {}, continuing to next repository. Error message: {}",
268                            targetRepository.getId(), path, e.getMessage(), e );
269             }
270 
271         }
272 
273         log.debug( "Exhausted all target repositories, resource {} not found.", path );
274 
275         return null;
276     }
277 
278     @Override
279     public ProxyFetchResult fetchMetadataFromProxies( ManagedRepository repository, String logicalPath )
280     {
281         StorageAsset localFile = repository.getAsset( logicalPath );
282 
283         Properties requestProperties = new Properties();
284         requestProperties.setProperty( "filetype", "metadata" );
285         boolean metadataNeedsUpdating = false;
286         long originalTimestamp = getLastModified( localFile );
287 
288         List<ProxyConnector> connectors = new ArrayList<>( getProxyConnectors( repository ) );
289         for ( ProxyConnector connector : connectors )
290         {
291             if ( !connector.isEnabled() )
292             {
293                 continue;
294             }
295 
296             RemoteRepository targetRepository = connector.getTargetRepository();
297 
298             StorageAsset localRepoFile = toLocalRepoFile( repository, targetRepository.getContent(), logicalPath );
299             long originalMetadataTimestamp = getLastModified( localRepoFile );
300 
301             try
302             {
303                 transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties,
304                               true );
305 
306                 if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
307                 {
308                     metadataNeedsUpdating = true;
309                 }
310             }
311             catch ( NotFoundException e )
312             {
313 
314                 log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath,
315                            targetRepository.getId(), e );
316 
317             }
318             catch ( NotModifiedException e )
319             {
320 
321                 log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath,
322                            targetRepository.getId(), e );
323 
324             }
325             catch ( ProxyException e )
326             {
327                 log.warn(
328                     "Transfer error from repository {} for versioned Metadata {}, continuing to next repository. Error message: {}",
329                     targetRepository.getId(), logicalPath, e.getMessage() );
330                 log.debug( "Full stack trace", e );
331             }
332         }
333 
334         if ( hasBeenUpdated( localFile, originalTimestamp ) )
335         {
336             metadataNeedsUpdating = true;
337         }
338 
339         if ( metadataNeedsUpdating || !localFile.exists())
340         {
341             try
342             {
343                 metadataTools.updateMetadata( repository.getContent(), logicalPath );
344             }
345             catch ( RepositoryMetadataException e )
346             {
347                 log.warn( "Unable to update metadata {}:{}", localFile.getPath(), e.getMessage(), e );
348             }
349 
350         }
351 
352         if ( fileExists( localFile ) )
353         {
354             return new ProxyFetchResult( localFile, metadataNeedsUpdating );
355         }
356 
357         return new ProxyFetchResult( null, false );
358     }
359 
360     private long getLastModified(StorageAsset file )
361     {
362         if ( !file.exists() || file.isContainer() )
363         {
364             return 0;
365         }
366 
367         return file.getModificationTime().toEpochMilli();
368     }
369 
370     private boolean hasBeenUpdated(StorageAsset file, long originalLastModified )
371     {
372         if ( !file.exists() || file.isContainer() )
373         {
374             return false;
375         }
376 
377         long currentLastModified = getLastModified( file );
378         return ( currentLastModified > originalLastModified );
379     }
380 
381     private StorageAsset toLocalRepoFile( ManagedRepository repository, RemoteRepositoryContent targetRepository,
382                                           String targetPath )
383     {
384         String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
385         return repository.getAsset( repoPath );
386     }
387 
388     /**
389      * Test if the provided ManagedRepositoryContent has any proxies configured for it.
390      * @param repository
391      */
392     @Override
393     public boolean hasProxies( ManagedRepository repository )
394     {
395         synchronized ( this.proxyConnectorMap )
396         {
397             return this.proxyConnectorMap.containsKey( repository.getId() );
398         }
399     }
400 
401     private StorageAsset toLocalFile(ManagedRepository repository, ArtifactReference artifact )
402     {
403         return repository.getContent().toFile( artifact );
404     }
405 
406     /**
407      * Simple method to test if the file exists on the local disk.
408      *
409      * @param file the file to test. (may be null)
410      * @return true if file exists. false if the file param is null, doesn't exist, or is not of type File.
411      */
412     private boolean fileExists( StorageAsset file )
413     {
414         if ( file == null )
415         {
416             return false;
417         }
418 
419         if ( !file.exists())
420         {
421             return false;
422         }
423 
424         return !file.isContainer();
425     }
426 
427     /**
428      * Perform the transfer of the file.
429      *
430      * @param connector         the connector configuration to use.
431      * @param remoteRepository  the remote repository get the resource from.
432      * @param remotePath        the path in the remote repository to the resource to get.
433      * @param repository        the managed repository that will hold the file
434      * @param resource          the path relative to the repository storage where the file should be downloaded to
435      * @param requestProperties the request properties to utilize for policy handling.
436      * @param executeConsumers  whether to execute the consumers after proxying
437      * @return the local file that was downloaded, or null if not downloaded.
438      * @throws NotFoundException    if the file was not found on the remote repository.
439      * @throws NotModifiedException if the localFile was present, and the resource was present on remote repository, but
440      *                              the remote resource is not newer than the local File.
441      * @throws ProxyException       if transfer was unsuccessful.
442      */
443     protected StorageAsset transferFile( ProxyConnector connector, RemoteRepository remoteRepository, String remotePath,
444                                          ManagedRepository repository, StorageAsset resource, Properties requestProperties,
445                                          boolean executeConsumers )
446         throws ProxyException, NotModifiedException
447     {
448         String url = null;
449         try
450         {
451             url = remoteRepository.getLocation().toURL().toString();
452         }
453         catch ( MalformedURLException e )
454         {
455             throw new ProxyException( e.getMessage(), e );
456         }
457         if ( !url.endsWith( "/" ) )
458         {
459             url = url + "/";
460         }
461         url = url + remotePath;
462         requestProperties.setProperty( "url", url );
463 
464         // Is a whitelist defined?
465         if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
466         {
467             // Path must belong to whitelist.
468             if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
469             {
470                 log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).",
471                            remotePath, remoteRepository.getId() );
472                 return null;
473             }
474         }
475 
476         // Is target path part of blacklist?
477         if ( matchesPattern( remotePath, connector.getBlacklist() ) )
478         {
479             log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath,
480                        remoteRepository.getId() );
481             return null;
482         }
483 
484         // Handle pre-download policy
485         try
486         {
487             validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
488         }
489         catch ( PolicyViolationException e )
490         {
491             String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
492             if ( resource.exists() )
493             {
494                 log.debug( "{} : using already present local file.", emsg );
495                 return resource;
496             }
497 
498             log.debug( emsg );
499             return null;
500         }
501 
502         Path workingDirectory = createWorkingDirectory( repository );
503         FilesystemStorage tmpStorage = null;
504         try
505         {
506             tmpStorage = new FilesystemStorage( workingDirectory, fileLockManager );
507         }
508         catch ( IOException e )
509         {
510             throw new ProxyException( "Could not create tmp storage" );
511         }
512         StorageAsset tmpResource = tmpStorage.getAsset( resource.getName( ) );
513         StorageAssetsset.html#StorageAsset">StorageAsset[] tmpChecksumFiles = new StorageAsset[checksumAlgorithms.size()];
514         for(int i=0; i<checksumAlgorithms.size(); i++) {
515             ChecksumAlgorithm alg = checksumAlgorithms.get( i );
516             tmpChecksumFiles[i] = tmpStorage.getAsset( resource.getName() + "." + alg.getDefaultExtension() );
517         }
518 
519         try
520         {
521 
522             transferResources( connector, remoteRepository, tmpResource,tmpChecksumFiles , url, remotePath,
523                 resource, workingDirectory, repository );
524 
525             // Handle post-download policies.
526             try
527             {
528                 validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
529             }
530             catch ( PolicyViolationException e )
531             {
532                 log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() );
533                 executeConsumers = false;
534                 if ( !fileExists( tmpResource ) )
535                 {
536                     resource = null;
537                 }
538             }
539 
540             if ( resource != null )
541             {
542                 synchronized ( resource.getPath().intern() )
543                 {
544                     StorageAsset directory = resource.getParent();
545                     for (int i=0; i<tmpChecksumFiles.length; i++) {
546                         moveFileIfExists( tmpChecksumFiles[i], directory );
547                     }
548                     moveFileIfExists( tmpResource, directory );
549                 }
550             }
551         }
552         finally
553         {
554             org.apache.archiva.common.utils.FileUtils.deleteQuietly( workingDirectory );
555         }
556 
557         if ( executeConsumers )
558         {
559             // Just-in-time update of the index and database by executing the consumers for this artifact
560             //consumers.executeConsumers( connector.getSourceRepository().getRepository(), resource );
561             queueRepositoryTask( connector.getSourceRepository().getId(), resource );
562         }
563 
564         return resource;
565     }
566 
567     protected abstract void transferResources( ProxyConnector connector, RemoteRepository remoteRepository,
568                                                StorageAssetorage/StorageAsset.html#StorageAsset">StorageAsset../org/apache/archiva/repository/storage/StorageAsset.html#StorageAsset">StorageAsset tmpResource, StorageAssetorage/StorageAsset.html#StorageAsset">StorageAsset[] checksumFiles, String url, String remotePath, StorageAsset resource, Path workingDirectory,
569                                                ManagedRepository repository ) throws ProxyException;
570 
571     private void queueRepositoryTask(String repositoryId, StorageAsset localFile )
572     {
573         RepositoryTaskitory/model/RepositoryTask.html#RepositoryTask">RepositoryTask task = new RepositoryTask();
574         task.setRepositoryId( repositoryId );
575         task.setResourceFile( localFile );
576         task.setUpdateRelatedArtifacts( true );
577         task.setScanAll( true );
578 
579         try
580         {
581             scheduler.queueTask( task );
582         }
583         catch ( TaskQueueException e )
584         {
585             log.error( "Unable to queue repository task to execute consumers on resource file ['{}"
586                            + "'].", localFile.getName() );
587         }
588     }
589 
590     /**
591      * Moves the file into repository location if it exists
592      *
593      * @param fileToMove this could be either the main artifact, sha1 or md5 checksum file.
594      * @param directory  directory to write files to
595      */
596     private void moveFileIfExists( StorageAsset/../org/apache/archiva/repository/storage/StorageAsset.html#StorageAsset">StorageAsset fileToMove, StorageAsset directory )
597         throws ProxyException
598     {
599         if ( fileToMove != null && fileToMove.exists() )
600         {
601             StorageAsset newLocation = directory.getStorage().getAsset( directory.getPath()+ "/" + fileToMove.getName());
602             moveTempToTarget( fileToMove, newLocation );
603         }
604     }
605 
606     /**
607      * Apply the policies.
608      *
609      * @param policies  the map of policies to execute. (Map of String policy keys, to {@link DownloadPolicy} objects)
610      * @param settings  the map of settings for the policies to execute. (Map of String policy keys, to String policy
611      *                  setting)
612      * @param request   the request properties (utilized by the {@link DownloadPolicy#applyPolicy(PolicyOption, Properties, StorageAsset)}
613      *                  )
614      * @param localFile the local file (utilized by the {@link DownloadPolicy#applyPolicy(PolicyOption, Properties, StorageAsset)})
615      * @throws PolicyViolationException
616      */
617     private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<Policy, PolicyOption> settings,
618                                    Properties request, StorageAsset localFile )
619         throws PolicyViolationException
620     {
621         for ( Map.Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
622         {
623             // olamy with spring rolehint is now downloadPolicy#hint
624             // so substring after last # to get the hint as with plexus
625             String key = entry.getValue( ).getId( );
626             DownloadPolicy policy = entry.getValue();
627             PolicyOption option = settings.containsKey(policy ) ? settings.get(policy) : policy.getDefaultOption();
628 
629             log.debug( "Applying [{}] policy with [{}]", key, option );
630             try
631             {
632                 policy.applyPolicy( option, request, localFile );
633             }
634             catch ( PolicyConfigurationException e )
635             {
636                 log.error( e.getMessage(), e );
637             }
638         }
639     }
640 
641     private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<Policy, PolicyOption> settings,
642                                    Properties request, ArtifactReference artifact, RemoteRepositoryContent content,
643                                    StorageAsset localFile, Exception exception, Map<String, Exception> previousExceptions )
644         throws ProxyDownloadException
645     {
646         boolean process = true;
647         for ( Map.Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
648         {
649 
650             // olamy with spring rolehint is now downloadPolicy#hint
651             // so substring after last # to get the hint as with plexus
652             String key = entry.getValue( ).getId( );
653             DownloadErrorPolicy policy = entry.getValue();
654             PolicyOption option = settings.containsKey( policy ) ? settings.get(policy) : policy.getDefaultOption();
655 
656             log.debug( "Applying [{}] policy with [{}]", key, option );
657             try
658             {
659                 // all policies must approve the exception, any can cancel
660                 process = policy.applyPolicy( option, request, localFile, exception, previousExceptions );
661                 if ( !process )
662                 {
663                     break;
664                 }
665             }
666             catch ( PolicyConfigurationException e )
667             {
668                 log.error( e.getMessage(), e );
669             }
670         }
671 
672         if ( process )
673         {
674             // if the exception was queued, don't throw it
675             if ( !previousExceptions.containsKey( content.getId() ) )
676             {
677                 throw new ProxyDownloadException(
678                     "An error occurred in downloading from the remote repository, and the policy is to fail immediately",
679                     content.getId(), exception );
680             }
681         }
682         else
683         {
684             // if the exception was queued, but cancelled, remove it
685             previousExceptions.remove( content.getId() );
686         }
687 
688         log.warn(
689             "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}",
690             content.getRepository().getId(), Keys.toKey( artifact ), exception.getMessage() );
691         log.debug( "Full stack trace", exception );
692     }
693 
694     /**
695      * Creates a working directory
696      *
697      * @param repository
698      * @return file location of working directory
699      */
700     private Path createWorkingDirectory( ManagedRepository repository )
701     {
702         try
703         {
704             return Files.createTempDirectory( "temp" );
705         }
706         catch ( IOException e )
707         {
708             throw new RuntimeException( e.getMessage(), e );
709         }
710 
711     }
712 
713     /**
714      * Used to move the temporary file to its real destination. This is patterned from the way WagonManager handles its
715      * downloaded files.
716      *
717      * @param temp   The completed download file
718      * @param target The final location of the downloaded file
719      * @throws ProxyException when the temp file cannot replace the target file
720      */
721     private void moveTempToTarget( StorageAsset/../../../org/apache/archiva/repository/storage/StorageAsset.html#StorageAsset">StorageAsset temp, StorageAsset target )
722         throws ProxyException
723     {
724 
725         try
726         {
727             StorageUtil.moveAsset( temp, target, true , StandardCopyOption.REPLACE_EXISTING);
728         }
729         catch ( IOException e )
730         {
731             log.error( "Move failed from {} to {}, trying copy.", temp, target );
732             try
733             {
734                 StorageUtil.copyAsset( temp, target, true );
735                 if (temp.exists()) {
736                     temp.getStorage( ).removeAsset( temp );
737                 }
738             }
739             catch ( IOException ex )
740             {
741                 log.error("Copy failed from {} to {}: ({}) {}", temp, target, e.getClass(), e.getMessage());
742                 throw new ProxyException("Could not move temp file "+temp.getPath()+" to target "+target.getPath()+": ("+e.getClass()+") "+e.getMessage(), e);
743             }
744         }
745     }
746 
747     /**
748      * Tests whitelist and blacklist patterns against path.
749      *
750      * @param path     the path to test.
751      * @param patterns the list of patterns to check.
752      * @return true if the path matches at least 1 pattern in the provided patterns list.
753      */
754     private boolean matchesPattern( String path, List<String> patterns )
755     {
756         if ( CollectionUtils.isEmpty( patterns ) )
757         {
758             return false;
759         }
760 
761         if ( !path.startsWith( "/" ) )
762         {
763             path = "/" + path;
764         }
765 
766         for ( String pattern : patterns )
767         {
768             if ( !pattern.startsWith( "/" ) )
769             {
770                 pattern = "/" + pattern;
771             }
772 
773             if ( SelectorUtils.matchPath( pattern, path, false ) )
774             {
775                 return true;
776             }
777         }
778 
779         return false;
780     }
781 
782     /**
783      * TODO: Ensure that list is correctly ordered based on configuration. See MRM-477
784      * @param repository
785      */
786     @Override
787     public List<ProxyConnector> getProxyConnectors( ManagedRepository repository )
788     {
789 
790         if ( !this.proxyConnectorMap.containsKey( repository.getId() ) )
791         {
792             return Collections.emptyList();
793         }
794         List<ProxyConnector> ret = new ArrayList<>( this.proxyConnectorMap.get( repository.getId() ) );
795 
796         Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
797         return ret;
798 
799     }
800 
801 
802     protected String addParameters(String path, RemoteRepository remoteRepository )
803     {
804         if ( remoteRepository.getExtraParameters().isEmpty() )
805         {
806             return path;
807         }
808 
809         boolean question = false;
810 
811         StringBuilder res = new StringBuilder( path == null ? "" : path );
812 
813         for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
814         {
815             if ( !question )
816             {
817                 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
818             }
819         }
820 
821         return res.toString();
822     }
823 
824     public void setArchivaConfiguration(ArchivaConfiguration archivaConfiguration )
825     {
826         this.archivaConfiguration = archivaConfiguration;
827     }
828 
829     public MetadataTools getMetadataTools()
830     {
831         return metadataTools;
832     }
833 
834     public void setMetadataTools(MetadataTools metadataTools )
835     {
836         this.metadataTools = metadataTools;
837     }
838 
839     public UrlFailureCache getUrlFailureCache()
840     {
841         return urlFailureCache;
842     }
843 
844     public void setUrlFailureCache(UrlFailureCache urlFailureCache )
845     {
846         this.urlFailureCache = urlFailureCache;
847     }
848 
849     public Map<String, PreDownloadPolicy> getPreDownloadPolicies()
850     {
851         return preDownloadPolicies;
852     }
853 
854     public void setPreDownloadPolicies(Map<String, PreDownloadPolicy> preDownloadPolicies )
855     {
856         this.preDownloadPolicies = preDownloadPolicies;
857     }
858 
859     public Map<String, PostDownloadPolicy> getPostDownloadPolicies()
860     {
861         return postDownloadPolicies;
862     }
863 
864     public void setPostDownloadPolicies(Map<String, PostDownloadPolicy> postDownloadPolicies )
865     {
866         this.postDownloadPolicies = postDownloadPolicies;
867     }
868 
869     public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies()
870     {
871         return downloadErrorPolicies;
872     }
873 
874     public void setDownloadErrorPolicies(Map<String, DownloadErrorPolicy> downloadErrorPolicies )
875     {
876         this.downloadErrorPolicies = downloadErrorPolicies;
877     }
878 
879     @Override
880     public void setNetworkProxies(Map<String, NetworkProxy> networkProxies ) {
881         this.networkProxyMap.clear();
882         this.networkProxyMap.putAll( networkProxies );
883     }
884 
885     @Override
886     public NetworkProxy getNetworkProxy(String id) {
887         return this.networkProxyMap.get(id);
888     }
889 
890     @Override
891     public Map<String, NetworkProxy> getNetworkProxies() {
892         return this.networkProxyMap;
893     }
894 
895     @Override
896     public abstract List<RepositoryType> supports();
897 
898     @Override
899     public void setPolicies( List<Policy> policyList )
900     {
901         preDownloadPolicies.clear();
902         postDownloadPolicies.clear();
903         downloadErrorPolicies.clear();
904         for (Policy policy : policyList) {
905             addPolicy( policy );
906         }
907     }
908 
909     void addPolicy(PreDownloadPolicy policy) {
910         preDownloadPolicies.put( policy.getId( ), policy );
911     }
912 
913     void addPolicy(PostDownloadPolicy policy) {
914         postDownloadPolicies.put( policy.getId( ), policy );
915     }
916     void addPolicy(DownloadErrorPolicy policy) {
917         downloadErrorPolicies.put( policy.getId( ), policy );
918     }
919 
920     @Override
921     public void addPolicy( Policy policy )
922     {
923         if (policy instanceof PreDownloadPolicy) {
924             addPolicy( (PreDownloadPolicy)policy );
925         } else if (policy instanceof PostDownloadPolicy) {
926             addPolicy( (PostDownloadPolicy) policy );
927         } else if (policy instanceof DownloadErrorPolicy) {
928             addPolicy( (DownloadErrorPolicy) policy );
929         } else {
930             log.warn( "Policy not known: {}, {}", policy.getId( ), policy.getClass( ).getName( ) );
931         }
932     }
933 
934     @Override
935     public void removePolicy( Policy policy )
936     {
937         final String id = policy.getId();
938         if (preDownloadPolicies.containsKey( id )) {
939             preDownloadPolicies.remove( id );
940         } else if (postDownloadPolicies.containsKey( id )) {
941             postDownloadPolicies.remove( id );
942         } else if (downloadErrorPolicies.containsKey( id )) {
943             downloadErrorPolicies.remove( id );
944         }
945     }
946 
947     @Override
948     public void addProxyConnector( ProxyConnector connector )
949     {
950         final String sourceId = connector.getSourceRepository( ).getId( );
951         List<ProxyConnector> connectors;
952         if (proxyConnectorMap.containsKey( sourceId )) {
953             connectors = proxyConnectorMap.get( sourceId );
954         } else {
955             connectors = new ArrayList<>( );
956             proxyConnectorMap.put( sourceId, connectors );
957         }
958         connectors.add( connector );
959     }
960 
961     @Override
962     public void setProxyConnectors( List<ProxyConnector> proxyConnectors )
963     {
964         proxyConnectorMap.clear();
965         for ( ProxyConnector connector : proxyConnectors )
966         {
967             addProxyConnector( connector );
968         }
969     }
970 }