Source code

001package org.apache.archiva.proxy;
002
003/*
004 * Licensed to the Apache Software Foundation (ASF) under one
005 * or more contributor license agreements.  See the NOTICE file
006 * distributed with this work for additional information
007 * regarding copyright ownership.  The ASF licenses this file
008 * to you under the Apache License, Version 2.0 (the
009 * "License"); you may not use this file except in compliance
010 * with the License.  You may obtain a copy of the License at
011 *
012 *  http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing,
015 * software distributed under the License is distributed on an
016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
017 * KIND, either express or implied.  See the License for the
018 * specific language governing permissions and limitations
019 * under the License.
020 */
021
022import org.apache.archiva.checksum.ChecksumAlgorithm;
023import org.apache.archiva.checksum.ChecksumUtil;
024import org.apache.archiva.common.filelock.FileLockManager;
025import org.apache.archiva.configuration.ArchivaConfiguration;
026import org.apache.archiva.configuration.ProxyConnectorConfiguration;
027import org.apache.archiva.configuration.ProxyConnectorRuleConfiguration;
028import org.apache.archiva.model.ArtifactReference;
029import org.apache.archiva.model.Keys;
030import org.apache.archiva.policies.DownloadErrorPolicy;
031import org.apache.archiva.policies.DownloadPolicy;
032import org.apache.archiva.policies.Policy;
033import org.apache.archiva.policies.PolicyConfigurationException;
034import org.apache.archiva.policies.PolicyOption;
035import org.apache.archiva.policies.PolicyViolationException;
036import org.apache.archiva.policies.PostDownloadPolicy;
037import org.apache.archiva.policies.PreDownloadPolicy;
038import org.apache.archiva.policies.ProxyDownloadException;
039import org.apache.archiva.policies.urlcache.UrlFailureCache;
040import org.apache.archiva.proxy.model.NetworkProxy;
041import org.apache.archiva.proxy.model.ProxyConnector;
042import org.apache.archiva.proxy.model.ProxyFetchResult;
043import org.apache.archiva.proxy.model.RepositoryProxyHandler;
044import org.apache.archiva.components.taskqueue.TaskQueueException;
045import org.apache.archiva.repository.ManagedRepository;
046import org.apache.archiva.repository.RemoteRepository;
047import org.apache.archiva.repository.RemoteRepositoryContent;
048import org.apache.archiva.repository.RepositoryType;
049import org.apache.archiva.repository.metadata.base.MetadataTools;
050import org.apache.archiva.repository.metadata.RepositoryMetadataException;
051import org.apache.archiva.repository.storage.FilesystemStorage;
052import org.apache.archiva.repository.storage.StorageAsset;
053import org.apache.archiva.repository.storage.StorageUtil;
054import org.apache.archiva.scheduler.ArchivaTaskScheduler;
055import org.apache.archiva.scheduler.repository.model.RepositoryTask;
056import org.apache.commons.collections4.CollectionUtils;
057import org.apache.commons.io.FilenameUtils;
058import org.apache.commons.lang3.StringUtils;
059import org.apache.commons.lang3.SystemUtils;
060import org.apache.tools.ant.types.selectors.SelectorUtils;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063import org.slf4j.MarkerFactory;
064
065import javax.annotation.PostConstruct;
066import javax.inject.Inject;
067import javax.inject.Named;
068import java.io.IOException;
069import java.net.MalformedURLException;
070import java.nio.file.Files;
071import java.nio.file.Path;
072import java.nio.file.StandardCopyOption;
073import java.util.ArrayList;
074import java.util.Collections;
075import java.util.HashMap;
076import java.util.LinkedHashMap;
077import java.util.List;
078import java.util.Map;
079import java.util.Properties;
080import java.util.concurrent.ConcurrentHashMap;
081import java.util.concurrent.ConcurrentMap;
082
083public abstract class DefaultRepositoryProxyHandler implements RepositoryProxyHandler {
084
085    protected Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyHandler.class );
086    @Inject
087    protected UrlFailureCache urlFailureCache;
088
089    @Inject
090    @Named(value = "metadataTools#default")
091    private MetadataTools metadataTools;
092
093    private Map<String, PreDownloadPolicy> preDownloadPolicies = new HashMap<>(  );
094    private Map<String, PostDownloadPolicy> postDownloadPolicies = new HashMap<>(  );
095    private Map<String, DownloadErrorPolicy> downloadErrorPolicies = new HashMap<>(  );
096    private ConcurrentMap<String, List<ProxyConnector>> proxyConnectorMap = new ConcurrentHashMap<>();
097
098    @Inject
099    @Named(value = "archivaTaskScheduler#repository")
100    private ArchivaTaskScheduler<RepositoryTask> scheduler;
101
102    @Inject
103    private ArchivaConfiguration archivaConfiguration;
104
105    @Inject
106    @Named(value = "fileLockManager#default")
107    private FileLockManager fileLockManager;
108
109    private Map<String, NetworkProxy> networkProxyMap = new ConcurrentHashMap<>();
110    private List<ChecksumAlgorithm> checksumAlgorithms;
111
112    @PostConstruct
113    public void initialize()
114    {
115        checksumAlgorithms = ChecksumUtil.getAlgorithms(archivaConfiguration.getConfiguration().getArchivaRuntimeConfiguration().getChecksumTypes());
116    }
117
118    private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules(String sourceRepository,
119                                                                          String targetRepository,
120                                                                          List<ProxyConnectorRuleConfiguration> all )
121    {
122        List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = new ArrayList<>();
123
124        for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all )
125        {
126            for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() )
127            {
128                if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals(
129                    targetRepository, proxyConnector.getTargetRepoId() ) )
130                {
131                    proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration );
132                }
133            }
134        }
135
136        return proxyConnectorRuleConfigurations;
137    }
138
139    @Override
140    public StorageAsset fetchFromProxies( ManagedRepository repository, ArtifactReference artifact )
141        throws ProxyDownloadException
142    {
143        StorageAsset localFile = toLocalFile( repository, artifact );
144
145        Properties requestProperties = new Properties();
146        requestProperties.setProperty( "filetype", "artifact" );
147        requestProperties.setProperty( "version", artifact.getVersion() );
148        requestProperties.setProperty( "managedRepositoryId", repository.getId() );
149
150        List<ProxyConnector> connectors = getProxyConnectors( repository );
151        Map<String, Exception> previousExceptions = new LinkedHashMap<>();
152        for ( ProxyConnector connector : connectors )
153        {
154            if ( !connector.isEnabled() )
155            {
156                continue;
157            }
158
159            RemoteRepository targetRepository = connector.getTargetRepository();
160            requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
161
162            String targetPath = targetRepository.getContent().toPath( artifact );
163
164            if ( SystemUtils.IS_OS_WINDOWS )
165            {
166                // toPath use system PATH_SEPARATOR so on windows url are \ which doesn't work very well :-)
167                targetPath = FilenameUtils.separatorsToUnix( targetPath );
168            }
169
170            try
171            {
172                StorageAsset downloadedFile =
173                    transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
174                                  true );
175
176                if ( fileExists(downloadedFile) )
177                {
178                    log.debug( "Successfully transferred: {}", downloadedFile.getPath() );
179                    return downloadedFile;
180                }
181            }
182            catch ( NotFoundException e )
183            {
184                log.debug( "Artifact {} not found on repository \"{}\".", Keys.toKey( artifact ),
185                           targetRepository.getId() );
186            }
187            catch ( NotModifiedException e )
188            {
189                log.debug( "Artifact {} not updated on repository \"{}\".", Keys.toKey( artifact ),
190                           targetRepository.getId() );
191            }
192            catch ( ProxyException e )
193            {
194                validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
195                                  targetRepository.getContent(), localFile, e, previousExceptions );
196            }
197        }
198
199        if ( !previousExceptions.isEmpty() )
200        {
201            throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
202                                              previousExceptions );
203        }
204
205        log.debug( "Exhausted all target repositories, artifact {} not found.", Keys.toKey( artifact ) );
206
207        return null;
208    }
209
210    @Override
211    public StorageAsset fetchFromProxies( ManagedRepository repository, String path )
212    {
213        StorageAsset localFile = repository.getAsset( path );
214
215        // no update policies for these paths
216        if ( localFile.exists() )
217        {
218            return null;
219        }
220
221        Properties requestProperties = new Properties();
222        requestProperties.setProperty( "filetype", "resource" );
223        requestProperties.setProperty( "managedRepositoryId", repository.getId() );
224
225        List<ProxyConnector> connectors = getProxyConnectors( repository );
226        for ( ProxyConnector connector : connectors )
227        {
228            if ( !connector.isEnabled() )
229            {
230                continue;
231            }
232
233            RemoteRepository targetRepository = connector.getTargetRepository();
234            requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
235
236            String targetPath = path;
237
238            try
239            {
240                StorageAsset downloadedFile =
241                    transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
242                                  false );
243
244                if ( fileExists( downloadedFile ) )
245                {
246                    log.debug( "Successfully transferred: {}", downloadedFile.getPath() );
247                    return downloadedFile;
248                }
249            }
250            catch ( NotFoundException e )
251            {
252                log.debug( "Resource {} not found on repository \"{}\".", path,
253                           targetRepository.getId() );
254            }
255            catch ( NotModifiedException e )
256            {
257                log.debug( "Resource {} not updated on repository \"{}\".", path,
258                           targetRepository.getId() );
259            }
260            catch ( ProxyException e )
261            {
262                log.warn(
263                    "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
264                    targetRepository.getId(), path, e.getMessage() );
265                log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
266                           "Transfer error from repository \"{}"
267                               + "\" for resource {}, continuing to next repository. Error message: {}",
268                           targetRepository.getId(), path, e.getMessage(), e );
269            }
270
271        }
272
273        log.debug( "Exhausted all target repositories, resource {} not found.", path );
274
275        return null;
276    }
277
278    @Override
279    public ProxyFetchResult fetchMetadataFromProxies( ManagedRepository repository, String logicalPath )
280    {
281        StorageAsset localFile = repository.getAsset( logicalPath );
282
283        Properties requestProperties = new Properties();
284        requestProperties.setProperty( "filetype", "metadata" );
285        boolean metadataNeedsUpdating = false;
286        long originalTimestamp = getLastModified( localFile );
287
288        List<ProxyConnector> connectors = new ArrayList<>( getProxyConnectors( repository ) );
289        for ( ProxyConnector connector : connectors )
290        {
291            if ( !connector.isEnabled() )
292            {
293                continue;
294            }
295
296            RemoteRepository targetRepository = connector.getTargetRepository();
297
298            StorageAsset localRepoFile = toLocalRepoFile( repository, targetRepository.getContent(), logicalPath );
299            long originalMetadataTimestamp = getLastModified( localRepoFile );
300
301            try
302            {
303                transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties,
304                              true );
305
306                if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
307                {
308                    metadataNeedsUpdating = true;
309                }
310            }
311            catch ( NotFoundException e )
312            {
313
314                log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath,
315                           targetRepository.getId(), e );
316
317            }
318            catch ( NotModifiedException e )
319            {
320
321                log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath,
322                           targetRepository.getId(), e );
323
324            }
325            catch ( ProxyException e )
326            {
327                log.warn(
328                    "Transfer error from repository {} for versioned Metadata {}, continuing to next repository. Error message: {}",
329                    targetRepository.getId(), logicalPath, e.getMessage() );
330                log.debug( "Full stack trace", e );
331            }
332        }
333
334        if ( hasBeenUpdated( localFile, originalTimestamp ) )
335        {
336            metadataNeedsUpdating = true;
337        }
338
339        if ( metadataNeedsUpdating || !localFile.exists())
340        {
341            try
342            {
343                metadataTools.updateMetadata( repository.getContent(), logicalPath );
344            }
345            catch ( RepositoryMetadataException e )
346            {
347                log.warn( "Unable to update metadata {}:{}", localFile.getPath(), e.getMessage(), e );
348            }
349
350        }
351
352        if ( fileExists( localFile ) )
353        {
354            return new ProxyFetchResult( localFile, metadataNeedsUpdating );
355        }
356
357        return new ProxyFetchResult( null, false );
358    }
359
360    private long getLastModified(StorageAsset file )
361    {
362        if ( !file.exists() || file.isContainer() )
363        {
364            return 0;
365        }
366
367        return file.getModificationTime().toEpochMilli();
368    }
369
370    private boolean hasBeenUpdated(StorageAsset file, long originalLastModified )
371    {
372        if ( !file.exists() || file.isContainer() )
373        {
374            return false;
375        }
376
377        long currentLastModified = getLastModified( file );
378        return ( currentLastModified > originalLastModified );
379    }
380
381    private StorageAsset toLocalRepoFile( ManagedRepository repository, RemoteRepositoryContent targetRepository,
382                                          String targetPath )
383    {
384        String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
385        return repository.getAsset( repoPath );
386    }
387
388    /**
389     * Test if the provided ManagedRepositoryContent has any proxies configured for it.
390     * @param repository
391     */
392    @Override
393    public boolean hasProxies( ManagedRepository repository )
394    {
395        synchronized ( this.proxyConnectorMap )
396        {
397            return this.proxyConnectorMap.containsKey( repository.getId() );
398        }
399    }
400
401    private StorageAsset toLocalFile(ManagedRepository repository, ArtifactReference artifact )
402    {
403        return repository.getContent().toFile( artifact );
404    }
405
406    /**
407     * Simple method to test if the file exists on the local disk.
408     *
409     * @param file the file to test. (may be null)
410     * @return true if file exists. false if the file param is null, doesn't exist, or is not of type File.
411     */
412    private boolean fileExists( StorageAsset file )
413    {
414        if ( file == null )
415        {
416            return false;
417        }
418
419        if ( !file.exists())
420        {
421            return false;
422        }
423
424        return !file.isContainer();
425    }
426
427    /**
428     * Perform the transfer of the file.
429     *
430     * @param connector         the connector configuration to use.
431     * @param remoteRepository  the remote repository get the resource from.
432     * @param remotePath        the path in the remote repository to the resource to get.
433     * @param repository        the managed repository that will hold the file
434     * @param resource          the path relative to the repository storage where the file should be downloaded to
435     * @param requestProperties the request properties to utilize for policy handling.
436     * @param executeConsumers  whether to execute the consumers after proxying
437     * @return the local file that was downloaded, or null if not downloaded.
438     * @throws NotFoundException    if the file was not found on the remote repository.
439     * @throws NotModifiedException if the localFile was present, and the resource was present on remote repository, but
440     *                              the remote resource is not newer than the local File.
441     * @throws ProxyException       if transfer was unsuccessful.
442     */
443    protected StorageAsset transferFile( ProxyConnector connector, RemoteRepository remoteRepository, String remotePath,
444                                         ManagedRepository repository, StorageAsset resource, Properties requestProperties,
445                                         boolean executeConsumers )
446        throws ProxyException, NotModifiedException
447    {
448        String url = null;
449        try
450        {
451            url = remoteRepository.getLocation().toURL().toString();
452        }
453        catch ( MalformedURLException e )
454        {
455            throw new ProxyException( e.getMessage(), e );
456        }
457        if ( !url.endsWith( "/" ) )
458        {
459            url = url + "/";
460        }
461        url = url + remotePath;
462        requestProperties.setProperty( "url", url );
463
464        // Is a whitelist defined?
465        if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
466        {
467            // Path must belong to whitelist.
468            if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
469            {
470                log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).",
471                           remotePath, remoteRepository.getId() );
472                return null;
473            }
474        }
475
476        // Is target path part of blacklist?
477        if ( matchesPattern( remotePath, connector.getBlacklist() ) )
478        {
479            log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath,
480                       remoteRepository.getId() );
481            return null;
482        }
483
484        // Handle pre-download policy
485        try
486        {
487            validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
488        }
489        catch ( PolicyViolationException e )
490        {
491            String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
492            if ( resource.exists() )
493            {
494                log.debug( "{} : using already present local file.", emsg );
495                return resource;
496            }
497
498            log.debug( emsg );
499            return null;
500        }
501
502        Path workingDirectory = createWorkingDirectory( repository );
503        FilesystemStorage tmpStorage = null;
504        try
505        {
506            tmpStorage = new FilesystemStorage( workingDirectory, fileLockManager );
507        }
508        catch ( IOException e )
509        {
510            throw new ProxyException( "Could not create tmp storage" );
511        }
512        StorageAsset tmpResource = tmpStorage.getAsset( resource.getName( ) );
513        StorageAsset[] tmpChecksumFiles = new StorageAsset[checksumAlgorithms.size()];
514        for(int i=0; i<checksumAlgorithms.size(); i++) {
515            ChecksumAlgorithm alg = checksumAlgorithms.get( i );
516            tmpChecksumFiles[i] = tmpStorage.getAsset( resource.getName() + "." + alg.getDefaultExtension() );
517        }
518
519        try
520        {
521
522            transferResources( connector, remoteRepository, tmpResource,tmpChecksumFiles , url, remotePath,
523                resource, workingDirectory, repository );
524
525            // Handle post-download policies.
526            try
527            {
528                validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
529            }
530            catch ( PolicyViolationException e )
531            {
532                log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() );
533                executeConsumers = false;
534                if ( !fileExists( tmpResource ) )
535                {
536                    resource = null;
537                }
538            }
539
540            if ( resource != null )
541            {
542                synchronized ( resource.getPath().intern() )
543                {
544                    StorageAsset directory = resource.getParent();
545                    for (int i=0; i<tmpChecksumFiles.length; i++) {
546                        moveFileIfExists( tmpChecksumFiles[i], directory );
547                    }
548                    moveFileIfExists( tmpResource, directory );
549                }
550            }
551        }
552        finally
553        {
554            org.apache.archiva.common.utils.FileUtils.deleteQuietly( workingDirectory );
555        }
556
557        if ( executeConsumers )
558        {
559            // Just-in-time update of the index and database by executing the consumers for this artifact
560            //consumers.executeConsumers( connector.getSourceRepository().getRepository(), resource );
561            queueRepositoryTask( connector.getSourceRepository().getId(), resource );
562        }
563
564        return resource;
565    }
566
567    protected abstract void transferResources( ProxyConnector connector, RemoteRepository remoteRepository,
568                                               StorageAsset tmpResource, StorageAsset[] checksumFiles, String url, String remotePath, StorageAsset resource, Path workingDirectory,
569                                               ManagedRepository repository ) throws ProxyException;
570
571    private void queueRepositoryTask(String repositoryId, StorageAsset localFile )
572    {
573        RepositoryTask task = new RepositoryTask();
574        task.setRepositoryId( repositoryId );
575        task.setResourceFile( localFile );
576        task.setUpdateRelatedArtifacts( true );
577        task.setScanAll( true );
578
579        try
580        {
581            scheduler.queueTask( task );
582        }
583        catch ( TaskQueueException e )
584        {
585            log.error( "Unable to queue repository task to execute consumers on resource file ['{}"
586                           + "'].", localFile.getName() );
587        }
588    }
589
590    /**
591     * Moves the file into repository location if it exists
592     *
593     * @param fileToMove this could be either the main artifact, sha1 or md5 checksum file.
594     * @param directory  directory to write files to
595     */
596    private void moveFileIfExists( StorageAsset fileToMove, StorageAsset directory )
597        throws ProxyException
598    {
599        if ( fileToMove != null && fileToMove.exists() )
600        {
601            StorageAsset newLocation = directory.getStorage().getAsset( directory.getPath()+ "/" + fileToMove.getName());
602            moveTempToTarget( fileToMove, newLocation );
603        }
604    }
605
606    /**
607     * Apply the policies.
608     *
609     * @param policies  the map of policies to execute. (Map of String policy keys, to {@link DownloadPolicy} objects)
610     * @param settings  the map of settings for the policies to execute. (Map of String policy keys, to String policy
611     *                  setting)
612     * @param request   the request properties (utilized by the {@link DownloadPolicy#applyPolicy(PolicyOption, Properties, StorageAsset)}
613     *                  )
614     * @param localFile the local file (utilized by the {@link DownloadPolicy#applyPolicy(PolicyOption, Properties, StorageAsset)})
615     * @throws PolicyViolationException
616     */
617    private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<Policy, PolicyOption> settings,
618                                   Properties request, StorageAsset localFile )
619        throws PolicyViolationException
620    {
621        for ( Map.Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
622        {
623            // olamy with spring rolehint is now downloadPolicy#hint
624            // so substring after last # to get the hint as with plexus
625            String key = entry.getValue( ).getId( );
626            DownloadPolicy policy = entry.getValue();
627            PolicyOption option = settings.containsKey(policy ) ? settings.get(policy) : policy.getDefaultOption();
628
629            log.debug( "Applying [{}] policy with [{}]", key, option );
630            try
631            {
632                policy.applyPolicy( option, request, localFile );
633            }
634            catch ( PolicyConfigurationException e )
635            {
636                log.error( e.getMessage(), e );
637            }
638        }
639    }
640
641    private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<Policy, PolicyOption> settings,
642                                   Properties request, ArtifactReference artifact, RemoteRepositoryContent content,
643                                   StorageAsset localFile, Exception exception, Map<String, Exception> previousExceptions )
644        throws ProxyDownloadException
645    {
646        boolean process = true;
647        for ( Map.Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
648        {
649
650            // olamy with spring rolehint is now downloadPolicy#hint
651            // so substring after last # to get the hint as with plexus
652            String key = entry.getValue( ).getId( );
653            DownloadErrorPolicy policy = entry.getValue();
654            PolicyOption option = settings.containsKey( policy ) ? settings.get(policy) : policy.getDefaultOption();
655
656            log.debug( "Applying [{}] policy with [{}]", key, option );
657            try
658            {
659                // all policies must approve the exception, any can cancel
660                process = policy.applyPolicy( option, request, localFile, exception, previousExceptions );
661                if ( !process )
662                {
663                    break;
664                }
665            }
666            catch ( PolicyConfigurationException e )
667            {
668                log.error( e.getMessage(), e );
669            }
670        }
671
672        if ( process )
673        {
674            // if the exception was queued, don't throw it
675            if ( !previousExceptions.containsKey( content.getId() ) )
676            {
677                throw new ProxyDownloadException(
678                    "An error occurred in downloading from the remote repository, and the policy is to fail immediately",
679                    content.getId(), exception );
680            }
681        }
682        else
683        {
684            // if the exception was queued, but cancelled, remove it
685            previousExceptions.remove( content.getId() );
686        }
687
688        log.warn(
689            "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}",
690            content.getRepository().getId(), Keys.toKey( artifact ), exception.getMessage() );
691        log.debug( "Full stack trace", exception );
692    }
693
694    /**
695     * Creates a working directory
696     *
697     * @param repository
698     * @return file location of working directory
699     */
700    private Path createWorkingDirectory( ManagedRepository repository )
701    {
702        try
703        {
704            return Files.createTempDirectory( "temp" );
705        }
706        catch ( IOException e )
707        {
708            throw new RuntimeException( e.getMessage(), e );
709        }
710
711    }
712
713    /**
714     * Used to move the temporary file to its real destination. This is patterned from the way WagonManager handles its
715     * downloaded files.
716     *
717     * @param temp   The completed download file
718     * @param target The final location of the downloaded file
719     * @throws ProxyException when the temp file cannot replace the target file
720     */
721    private void moveTempToTarget( StorageAsset temp, StorageAsset target )
722        throws ProxyException
723    {
724
725        try
726        {
727            StorageUtil.moveAsset( temp, target, true , StandardCopyOption.REPLACE_EXISTING);
728        }
729        catch ( IOException e )
730        {
731            log.error( "Move failed from {} to {}, trying copy.", temp, target );
732            try
733            {
734                StorageUtil.copyAsset( temp, target, true );
735                if (temp.exists()) {
736                    temp.getStorage( ).removeAsset( temp );
737                }
738            }
739            catch ( IOException ex )
740            {
741                log.error("Copy failed from {} to {}: ({}) {}", temp, target, e.getClass(), e.getMessage());
742                throw new ProxyException("Could not move temp file "+temp.getPath()+" to target "+target.getPath()+": ("+e.getClass()+") "+e.getMessage(), e);
743            }
744        }
745    }
746
747    /**
748     * Tests whitelist and blacklist patterns against path.
749     *
750     * @param path     the path to test.
751     * @param patterns the list of patterns to check.
752     * @return true if the path matches at least 1 pattern in the provided patterns list.
753     */
754    private boolean matchesPattern( String path, List<String> patterns )
755    {
756        if ( CollectionUtils.isEmpty( patterns ) )
757        {
758            return false;
759        }
760
761        if ( !path.startsWith( "/" ) )
762        {
763            path = "/" + path;
764        }
765
766        for ( String pattern : patterns )
767        {
768            if ( !pattern.startsWith( "/" ) )
769            {
770                pattern = "/" + pattern;
771            }
772
773            if ( SelectorUtils.matchPath( pattern, path, false ) )
774            {
775                return true;
776            }
777        }
778
779        return false;
780    }
781
782    /**
783     * TODO: Ensure that list is correctly ordered based on configuration. See MRM-477
784     * @param repository
785     */
786    @Override
787    public List<ProxyConnector> getProxyConnectors( ManagedRepository repository )
788    {
789
790        if ( !this.proxyConnectorMap.containsKey( repository.getId() ) )
791        {
792            return Collections.emptyList();
793        }
794        List<ProxyConnector> ret = new ArrayList<>( this.proxyConnectorMap.get( repository.getId() ) );
795
796        Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
797        return ret;
798
799    }
800
801
802    protected String addParameters(String path, RemoteRepository remoteRepository )
803    {
804        if ( remoteRepository.getExtraParameters().isEmpty() )
805        {
806            return path;
807        }
808
809        boolean question = false;
810
811        StringBuilder res = new StringBuilder( path == null ? "" : path );
812
813        for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
814        {
815            if ( !question )
816            {
817                res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
818            }
819        }
820
821        return res.toString();
822    }
823
824    public void setArchivaConfiguration(ArchivaConfiguration archivaConfiguration )
825    {
826        this.archivaConfiguration = archivaConfiguration;
827    }
828
829    public MetadataTools getMetadataTools()
830    {
831        return metadataTools;
832    }
833
834    public void setMetadataTools(MetadataTools metadataTools )
835    {
836        this.metadataTools = metadataTools;
837    }
838
839    public UrlFailureCache getUrlFailureCache()
840    {
841        return urlFailureCache;
842    }
843
844    public void setUrlFailureCache(UrlFailureCache urlFailureCache )
845    {
846        this.urlFailureCache = urlFailureCache;
847    }
848
849    public Map<String, PreDownloadPolicy> getPreDownloadPolicies()
850    {
851        return preDownloadPolicies;
852    }
853
854    public void setPreDownloadPolicies(Map<String, PreDownloadPolicy> preDownloadPolicies )
855    {
856        this.preDownloadPolicies = preDownloadPolicies;
857    }
858
859    public Map<String, PostDownloadPolicy> getPostDownloadPolicies()
860    {
861        return postDownloadPolicies;
862    }
863
864    public void setPostDownloadPolicies(Map<String, PostDownloadPolicy> postDownloadPolicies )
865    {
866        this.postDownloadPolicies = postDownloadPolicies;
867    }
868
869    public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies()
870    {
871        return downloadErrorPolicies;
872    }
873
874    public void setDownloadErrorPolicies(Map<String, DownloadErrorPolicy> downloadErrorPolicies )
875    {
876        this.downloadErrorPolicies = downloadErrorPolicies;
877    }
878
879    @Override
880    public void setNetworkProxies(Map<String, NetworkProxy> networkProxies ) {
881        this.networkProxyMap.clear();
882        this.networkProxyMap.putAll( networkProxies );
883    }
884
885    @Override
886    public NetworkProxy getNetworkProxy(String id) {
887        return this.networkProxyMap.get(id);
888    }
889
890    @Override
891    public Map<String, NetworkProxy> getNetworkProxies() {
892        return this.networkProxyMap;
893    }
894
895    @Override
896    public abstract List<RepositoryType> supports();
897
898    @Override
899    public void setPolicies( List<Policy> policyList )
900    {
901        preDownloadPolicies.clear();
902        postDownloadPolicies.clear();
903        downloadErrorPolicies.clear();
904        for (Policy policy : policyList) {
905            addPolicy( policy );
906        }
907    }
908
909    void addPolicy(PreDownloadPolicy policy) {
910        preDownloadPolicies.put( policy.getId( ), policy );
911    }
912
913    void addPolicy(PostDownloadPolicy policy) {
914        postDownloadPolicies.put( policy.getId( ), policy );
915    }
916    void addPolicy(DownloadErrorPolicy policy) {
917        downloadErrorPolicies.put( policy.getId( ), policy );
918    }
919
920    @Override
921    public void addPolicy( Policy policy )
922    {
923        if (policy instanceof PreDownloadPolicy) {
924            addPolicy( (PreDownloadPolicy)policy );
925        } else if (policy instanceof PostDownloadPolicy) {
926            addPolicy( (PostDownloadPolicy) policy );
927        } else if (policy instanceof DownloadErrorPolicy) {
928            addPolicy( (DownloadErrorPolicy) policy );
929        } else {
930            log.warn( "Policy not known: {}, {}", policy.getId( ), policy.getClass( ).getName( ) );
931        }
932    }
933
934    @Override
935    public void removePolicy( Policy policy )
936    {
937        final String id = policy.getId();
938        if (preDownloadPolicies.containsKey( id )) {
939            preDownloadPolicies.remove( id );
940        } else if (postDownloadPolicies.containsKey( id )) {
941            postDownloadPolicies.remove( id );
942        } else if (downloadErrorPolicies.containsKey( id )) {
943            downloadErrorPolicies.remove( id );
944        }
945    }
946
947    @Override
948    public void addProxyConnector( ProxyConnector connector )
949    {
950        final String sourceId = connector.getSourceRepository( ).getId( );
951        List<ProxyConnector> connectors;
952        if (proxyConnectorMap.containsKey( sourceId )) {
953            connectors = proxyConnectorMap.get( sourceId );
954        } else {
955            connectors = new ArrayList<>( );
956            proxyConnectorMap.put( sourceId, connectors );
957        }
958        connectors.add( connector );
959    }
960
961    @Override
962    public void setProxyConnectors( List<ProxyConnector> proxyConnectors )
963    {
964        proxyConnectorMap.clear();
965        for ( ProxyConnector connector : proxyConnectors )
966        {
967            addProxyConnector( connector );
968        }
969    }
970}