This project has retired. For details please refer to its
Attic page.
1 package org.apache.archiva.proxy;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import org.apache.archiva.admin.model.RepositoryAdminException;
23 import org.apache.archiva.admin.model.beans.NetworkProxy;
24 import org.apache.archiva.admin.model.beans.ProxyConnectorRuleType;
25 import org.apache.archiva.admin.model.beans.RemoteRepository;
26 import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
27 import org.apache.archiva.common.filelock.FileLockException;
28 import org.apache.archiva.common.filelock.FileLockManager;
29 import org.apache.archiva.common.filelock.FileLockTimeoutException;
30 import org.apache.archiva.common.filelock.Lock;
31 import org.apache.archiva.configuration.ArchivaConfiguration;
32 import org.apache.archiva.configuration.Configuration;
33 import org.apache.archiva.configuration.ConfigurationNames;
34 import org.apache.archiva.configuration.NetworkProxyConfiguration;
35 import org.apache.archiva.configuration.ProxyConnectorConfiguration;
36 import org.apache.archiva.configuration.ProxyConnectorRuleConfiguration;
37 import org.apache.archiva.model.ArtifactReference;
38 import org.apache.archiva.model.Keys;
39 import org.apache.archiva.model.RepositoryURL;
40 import org.apache.archiva.policies.DownloadErrorPolicy;
41 import org.apache.archiva.policies.DownloadPolicy;
42 import org.apache.archiva.policies.PolicyConfigurationException;
43 import org.apache.archiva.policies.PolicyViolationException;
44 import org.apache.archiva.policies.PostDownloadPolicy;
45 import org.apache.archiva.policies.PreDownloadPolicy;
46 import org.apache.archiva.policies.ProxyDownloadException;
47 import org.apache.archiva.policies.urlcache.UrlFailureCache;
48 import org.apache.archiva.proxy.common.WagonFactory;
49 import org.apache.archiva.proxy.common.WagonFactoryException;
50 import org.apache.archiva.proxy.common.WagonFactoryRequest;
51 import org.apache.archiva.proxy.model.ProxyConnector;
52 import org.apache.archiva.proxy.model.ProxyFetchResult;
53 import org.apache.archiva.proxy.model.RepositoryProxyConnectors;
54 import org.apache.archiva.redback.components.registry.Registry;
55 import org.apache.archiva.redback.components.registry.RegistryListener;
56 import org.apache.archiva.redback.components.taskqueue.TaskQueueException;
57 import org.apache.archiva.repository.ManagedRepositoryContent;
58 import org.apache.archiva.repository.RemoteRepositoryContent;
59 import org.apache.archiva.repository.RepositoryContentFactory;
60 import org.apache.archiva.repository.RepositoryException;
61 import org.apache.archiva.repository.RepositoryNotFoundException;
62 import org.apache.archiva.repository.metadata.MetadataTools;
63 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
64 import org.apache.archiva.scheduler.ArchivaTaskScheduler;
65 import org.apache.archiva.scheduler.repository.model.RepositoryTask;
66 import org.apache.commons.collections.CollectionUtils;
67 import org.apache.commons.io.FileUtils;
68 import org.apache.commons.io.FilenameUtils;
69 import org.apache.commons.lang.StringUtils;
70 import org.apache.commons.lang.SystemUtils;
71 import org.apache.maven.wagon.ConnectionException;
72 import org.apache.maven.wagon.ResourceDoesNotExistException;
73 import org.apache.maven.wagon.Wagon;
74 import org.apache.maven.wagon.WagonException;
75 import org.apache.maven.wagon.authentication.AuthenticationException;
76 import org.apache.maven.wagon.authentication.AuthenticationInfo;
77 import org.apache.maven.wagon.proxy.ProxyInfo;
78 import org.apache.maven.wagon.repository.Repository;
79 import org.apache.tools.ant.types.selectors.SelectorUtils;
80 import org.slf4j.Logger;
81 import org.slf4j.LoggerFactory;
82 import org.slf4j.MarkerFactory;
83 import org.springframework.stereotype.Service;
84
85 import javax.annotation.PostConstruct;
86 import javax.inject.Inject;
87 import javax.inject.Named;
88 import java.io.File;
89 import java.io.IOException;
90 import java.nio.file.Files;
91 import java.util.ArrayList;
92 import java.util.Collections;
93 import java.util.LinkedHashMap;
94 import java.util.List;
95 import java.util.Map;
96 import java.util.Map.Entry;
97 import java.util.Properties;
98 import java.util.concurrent.ConcurrentHashMap;
99 import java.util.concurrent.ConcurrentMap;
100 import java.util.concurrent.TimeUnit;
101
102
103
104
105
106
107 @Service("repositoryProxyConnectors#default")
108 public class DefaultRepositoryProxyConnectors
109 implements RepositoryProxyConnectors, RegistryListener
110 {
111 private Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyConnectors.class );
112
113 @Inject
114 @Named(value = "archivaConfiguration#default")
115 private ArchivaConfiguration archivaConfiguration;
116
117 @Inject
118 @Named(value = "repositoryContentFactory#default")
119 private RepositoryContentFactory repositoryFactory;
120
121 @Inject
122 @Named(value = "metadataTools#default")
123 private MetadataTools metadataTools;
124
125 @Inject
126 private Map<String, PreDownloadPolicy> preDownloadPolicies;
127
128 @Inject
129 private Map<String, PostDownloadPolicy> postDownloadPolicies;
130
131 @Inject
132 private Map<String, DownloadErrorPolicy> downloadErrorPolicies;
133
134 @Inject
135 private UrlFailureCache urlFailureCache;
136
137 private ConcurrentMap<String, List<ProxyConnector>> proxyConnectorMap = new ConcurrentHashMap<>();
138
139 private ConcurrentMap<String, ProxyInfo> networkProxyMap = new ConcurrentHashMap<>();
140
141 @Inject
142 private WagonFactory wagonFactory;
143
144 @Inject
145 @Named(value = "archivaTaskScheduler#repository")
146 private ArchivaTaskScheduler scheduler;
147
148 @Inject
149 private NetworkProxyAdmin networkProxyAdmin;
150
151 @Inject
152 @Named(value = "fileLockManager#default")
153 private FileLockManager fileLockManager;
154
155 @PostConstruct
156 public void initialize()
157 {
158 initConnectorsAndNetworkProxies();
159 archivaConfiguration.addChangeListener( this );
160
161 }
162
163 @SuppressWarnings("unchecked")
164 private void initConnectorsAndNetworkProxies()
165 {
166
167 ProxyConnectorOrderComparator proxyOrderSorter = new ProxyConnectorOrderComparator();
168 this.proxyConnectorMap.clear();
169
170 Configuration configuration = archivaConfiguration.getConfiguration();
171
172 List<ProxyConnectorRuleConfiguration> allProxyConnectorRuleConfigurations =
173 configuration.getProxyConnectorRuleConfigurations();
174
175 List<ProxyConnectorConfiguration> proxyConfigs = configuration.getProxyConnectors();
176 for ( ProxyConnectorConfiguration proxyConfig : proxyConfigs )
177 {
178 String key = proxyConfig.getSourceRepoId();
179
180 try
181 {
182
183 ProxyConnector connector = new ProxyConnector();
184
185 connector.setSourceRepository(
186 repositoryFactory.getManagedRepositoryContent( proxyConfig.getSourceRepoId() ) );
187 connector.setTargetRepository(
188 repositoryFactory.getRemoteRepositoryContent( proxyConfig.getTargetRepoId() ) );
189
190 connector.setProxyId( proxyConfig.getProxyId() );
191 connector.setPolicies( proxyConfig.getPolicies() );
192 connector.setOrder( proxyConfig.getOrder() );
193 connector.setDisabled( proxyConfig.isDisabled() );
194
195
196 List<String> blacklist = new ArrayList<>( 0 );
197 if ( CollectionUtils.isNotEmpty( proxyConfig.getBlackListPatterns() ) )
198 {
199 blacklist.addAll( proxyConfig.getBlackListPatterns() );
200 }
201 connector.setBlacklist( blacklist );
202
203
204 List<String> whitelist = new ArrayList<>( 0 );
205 if ( CollectionUtils.isNotEmpty( proxyConfig.getWhiteListPatterns() ) )
206 {
207 whitelist.addAll( proxyConfig.getWhiteListPatterns() );
208 }
209 connector.setWhitelist( whitelist );
210
211 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations =
212 findProxyConnectorRules( connector.getSourceRepository().getId(),
213 connector.getTargetRepository().getId(),
214 allProxyConnectorRuleConfigurations );
215
216 if ( !proxyConnectorRuleConfigurations.isEmpty() )
217 {
218 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : proxyConnectorRuleConfigurations )
219 {
220 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
221 ProxyConnectorRuleType.BLACK_LIST.getRuleType() ) )
222 {
223 connector.getBlacklist().add( proxyConnectorRuleConfiguration.getPattern() );
224 }
225
226 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
227 ProxyConnectorRuleType.WHITE_LIST.getRuleType() ) )
228 {
229 connector.getWhitelist().add( proxyConnectorRuleConfiguration.getPattern() );
230 }
231 }
232 }
233
234
235 List<ProxyConnector> connectors = this.proxyConnectorMap.get( key );
236 if ( connectors == null )
237 {
238
239 connectors = new ArrayList<>( 1 );
240 }
241
242
243 connectors.add( connector );
244
245
246 Collections.sort( connectors, proxyOrderSorter );
247
248
249 this.proxyConnectorMap.put( key, connectors );
250 }
251 catch ( RepositoryNotFoundException e )
252 {
253 log.warn( "Unable to use proxy connector: {}", e.getMessage(), e );
254 }
255 catch ( RepositoryException e )
256 {
257 log.warn( "Unable to use proxy connector: {}", e.getMessage(), e );
258 }
259
260
261 }
262
263 this.networkProxyMap.clear();
264
265 List<NetworkProxyConfiguration> networkProxies = archivaConfiguration.getConfiguration().getNetworkProxies();
266 for ( NetworkProxyConfiguration networkProxyConfig : networkProxies )
267 {
268 String key = networkProxyConfig.getId();
269
270 ProxyInfo proxy = new ProxyInfo();
271
272 proxy.setType( networkProxyConfig.getProtocol() );
273 proxy.setHost( networkProxyConfig.getHost() );
274 proxy.setPort( networkProxyConfig.getPort() );
275 proxy.setUserName( networkProxyConfig.getUsername() );
276 proxy.setPassword( networkProxyConfig.getPassword() );
277
278 this.networkProxyMap.put( key, proxy );
279 }
280
281 }
282
283 private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules( String sourceRepository,
284 String targetRepository,
285 List<ProxyConnectorRuleConfiguration> all )
286 {
287 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = new ArrayList<>();
288
289 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all )
290 {
291 for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() )
292 {
293 if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals(
294 targetRepository, proxyConnector.getTargetRepoId() ) )
295 {
296 proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration );
297 }
298 }
299 }
300
301 return proxyConnectorRuleConfigurations;
302 }
303
304 @Override
305 public File fetchFromProxies( ManagedRepositoryContent repository, ArtifactReference artifact )
306 throws ProxyDownloadException
307 {
308 File localFile = toLocalFile( repository, artifact );
309
310 Properties requestProperties = new Properties();
311 requestProperties.setProperty( "filetype", "artifact" );
312 requestProperties.setProperty( "version", artifact.getVersion() );
313 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
314
315 List<ProxyConnector> connectors = getProxyConnectors( repository );
316 Map<String, Exception> previousExceptions = new LinkedHashMap<>();
317 for ( ProxyConnector connector : connectors )
318 {
319 if ( connector.isDisabled() )
320 {
321 continue;
322 }
323
324 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
325 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
326
327 String targetPath = targetRepository.toPath( artifact );
328
329 if ( SystemUtils.IS_OS_WINDOWS )
330 {
331
332 targetPath = FilenameUtils.separatorsToUnix( targetPath );
333 }
334
335 try
336 {
337 File downloadedFile =
338 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
339 true );
340
341 if ( fileExists( downloadedFile ) )
342 {
343 log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
344 return downloadedFile;
345 }
346 }
347 catch ( NotFoundException e )
348 {
349 log.debug( "Artifact {} not found on repository \"{}\".", Keys.toKey( artifact ),
350 targetRepository.getRepository().getId() );
351 }
352 catch ( NotModifiedException e )
353 {
354 log.debug( "Artifact {} not updated on repository \"{}\".", Keys.toKey( artifact ),
355 targetRepository.getRepository().getId() );
356 }
357 catch ( ProxyException | RepositoryAdminException e )
358 {
359 validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
360 targetRepository, localFile, e, previousExceptions );
361 }
362 }
363
364 if ( !previousExceptions.isEmpty() )
365 {
366 throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
367 previousExceptions );
368 }
369
370 log.debug( "Exhausted all target repositories, artifact {} not found.", Keys.toKey( artifact ) );
371
372 return null;
373 }
374
375 @Override
376 public File fetchFromProxies( ManagedRepositoryContent repository, String path )
377 {
378 File localFile = new File( repository.getRepoRoot(), path );
379
380
381 if ( localFile.exists() )
382 {
383 return null;
384 }
385
386 Properties requestProperties = new Properties();
387 requestProperties.setProperty( "filetype", "resource" );
388 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
389
390 List<ProxyConnector> connectors = getProxyConnectors( repository );
391 for ( ProxyConnector connector : connectors )
392 {
393 if ( connector.isDisabled() )
394 {
395 continue;
396 }
397
398 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
399 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
400
401 String targetPath = path;
402
403 try
404 {
405 File downloadedFile =
406 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
407 false );
408
409 if ( fileExists( downloadedFile ) )
410 {
411 log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
412 return downloadedFile;
413 }
414 }
415 catch ( NotFoundException e )
416 {
417 log.debug( "Resource {} not found on repository \"{}\".", path,
418 targetRepository.getRepository().getId() );
419 }
420 catch ( NotModifiedException e )
421 {
422 log.debug( "Resource {} not updated on repository \"{}\".", path,
423 targetRepository.getRepository().getId() );
424 }
425 catch ( ProxyException e )
426 {
427 log.warn(
428 "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
429 targetRepository.getRepository().getId(), path, e.getMessage() );
430 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
431 "Transfer error from repository \"" + targetRepository.getRepository().getId()
432 + "\" for resource " + path + ", continuing to next repository. Error message: {}",
433 e.getMessage(), e
434 );
435 }
436 catch ( RepositoryAdminException e )
437 {
438 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
439 "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
440 targetRepository.getRepository().getId(), path, e.getMessage(), e );
441 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ), "Full stack trace", e );
442 }
443 }
444
445 log.debug( "Exhausted all target repositories, resource {} not found.", path );
446
447 return null;
448 }
449
450 @Override
451 public ProxyFetchResult fetchMetadataFromProxies( ManagedRepositoryContent repository, String logicalPath )
452 {
453 File localFile = new File( repository.getRepoRoot(), logicalPath );
454
455 Properties requestProperties = new Properties();
456 requestProperties.setProperty( "filetype", "metadata" );
457 boolean metadataNeedsUpdating = false;
458 long originalTimestamp = getLastModified( localFile );
459
460 List<ProxyConnector> connectors = new ArrayList<>( getProxyConnectors( repository ) );
461 for ( ProxyConnector connector : connectors )
462 {
463 if ( connector.isDisabled() )
464 {
465 continue;
466 }
467
468 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
469
470 File localRepoFile = toLocalRepoFile( repository, targetRepository, logicalPath );
471 long originalMetadataTimestamp = getLastModified( localRepoFile );
472
473 try
474 {
475 transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties,
476 true );
477
478 if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
479 {
480 metadataNeedsUpdating = true;
481 }
482 }
483 catch ( NotFoundException e )
484 {
485
486 log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath,
487 targetRepository.getRepository().getId(), e );
488
489 }
490 catch ( NotModifiedException e )
491 {
492
493 log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath,
494 targetRepository.getRepository().getId(), e );
495
496 }
497 catch ( ProxyException | RepositoryAdminException e )
498 {
499 log.warn(
500 "Transfer error from repository {} for versioned Metadata {}, continuing to next repository. Error message: {}",
501 targetRepository.getRepository().getId(), logicalPath, e.getMessage() );
502 log.debug( "Full stack trace", e );
503 }
504 }
505
506 if ( hasBeenUpdated( localFile, originalTimestamp ) )
507 {
508 metadataNeedsUpdating = true;
509 }
510
511 if ( metadataNeedsUpdating || !localFile.exists() )
512 {
513 try
514 {
515 metadataTools.updateMetadata( repository, logicalPath );
516 }
517 catch ( RepositoryMetadataException e )
518 {
519 log.warn( "Unable to update metadata {}:{}", localFile.getAbsolutePath(), e.getMessage(), e );
520 }
521
522 }
523
524 if ( fileExists( localFile ) )
525 {
526 return new ProxyFetchResult( localFile, metadataNeedsUpdating );
527 }
528
529 return new ProxyFetchResult( null, false );
530 }
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547 protected void transferResources( ProxyConnector connector, RemoteRepositoryContent remoteRepository, File tmpMd5,
548 File tmpSha1, File tmpResource, String url, String remotePath, File resource,
549 File workingDirectory, ManagedRepositoryContent repository )
550 throws ProxyException, NotModifiedException, RepositoryAdminException
551 {
552 Wagon wagon = null;
553 try
554 {
555 RepositoryURL repoUrl = remoteRepository.getURL();
556 String protocol = repoUrl.getProtocol();
557 NetworkProxy networkProxy = null;
558 if ( StringUtils.isNotBlank( connector.getProxyId() ) )
559 {
560 networkProxy = networkProxyAdmin.getNetworkProxy( connector.getProxyId() );
561 }
562 WagonFactoryRequest wagonFactoryRequest = new WagonFactoryRequest( "wagon#" + protocol,
563 remoteRepository.getRepository().getExtraHeaders() ).networkProxy(
564 networkProxy );
565 wagon = wagonFactory.getWagon( wagonFactoryRequest );
566 if ( wagon == null )
567 {
568 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
569 }
570
571 if ( wagon == null )
572 {
573 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
574 }
575
576 boolean connected = connectToRepository( connector, wagon, remoteRepository );
577 if ( connected )
578 {
579 transferArtifact( wagon, remoteRepository, remotePath, repository, resource, workingDirectory,
580 tmpResource );
581
582
583
584
585 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".sha1",
586 tmpSha1 );
587 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".md5",
588 tmpMd5 );
589 }
590 }
591 catch ( NotFoundException e )
592 {
593 urlFailureCache.cacheFailure( url );
594 throw e;
595 }
596 catch ( NotModifiedException e )
597 {
598
599 throw e;
600 }
601 catch ( ProxyException e )
602 {
603 urlFailureCache.cacheFailure( url );
604 throw e;
605 }
606 catch ( WagonFactoryException e )
607 {
608 throw new ProxyException( e.getMessage(), e );
609 }
610 finally
611 {
612 if ( wagon != null )
613 {
614 try
615 {
616 wagon.disconnect();
617 }
618 catch ( ConnectionException e )
619 {
620 log.warn( "Unable to disconnect wagon.", e );
621 }
622 }
623 }
624 }
625
626 private void transferArtifact( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
627 ManagedRepositoryContent repository, File resource, File tmpDirectory,
628 File destFile )
629 throws ProxyException
630 {
631 transferSimpleFile( wagon, remoteRepository, remotePath, repository, resource, destFile );
632 }
633
634 private long getLastModified( File file )
635 {
636 if ( !file.exists() || !file.isFile() )
637 {
638 return 0;
639 }
640
641 return file.lastModified();
642 }
643
644 private boolean hasBeenUpdated( File file, long originalLastModified )
645 {
646 if ( !file.exists() || !file.isFile() )
647 {
648 return false;
649 }
650
651 long currentLastModified = getLastModified( file );
652 return ( currentLastModified > originalLastModified );
653 }
654
655 private File toLocalRepoFile( ManagedRepositoryContent repository, RemoteRepositoryContent targetRepository,
656 String targetPath )
657 {
658 String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
659 return new File( repository.getRepoRoot(), repoPath );
660 }
661
662
663
664
665 @Override
666 public boolean hasProxies( ManagedRepositoryContent repository )
667 {
668 synchronized ( this.proxyConnectorMap )
669 {
670 return this.proxyConnectorMap.containsKey( repository.getId() );
671 }
672 }
673
674 private File toLocalFile( ManagedRepositoryContent repository, ArtifactReference artifact )
675 {
676 return repository.toFile( artifact );
677 }
678
679
680
681
682
683
684
685 private boolean fileExists( File file )
686 {
687 if ( file == null )
688 {
689 return false;
690 }
691
692 if ( !file.exists() )
693 {
694 return false;
695 }
696
697 return file.isFile();
698 }
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716 private File transferFile( ProxyConnector connector, RemoteRepositoryContent remoteRepository, String remotePath,
717 ManagedRepositoryContent repository, File resource, Properties requestProperties,
718 boolean executeConsumers )
719 throws ProxyException, NotModifiedException, RepositoryAdminException
720 {
721 String url = remoteRepository.getURL().getUrl();
722 if ( !url.endsWith( "/" ) )
723 {
724 url = url + "/";
725 }
726 url = url + remotePath;
727 requestProperties.setProperty( "url", url );
728
729
730 if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
731 {
732
733 if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
734 {
735 log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).",
736 remotePath, remoteRepository.getRepository().getName() );
737 return null;
738 }
739 }
740
741
742 if ( matchesPattern( remotePath, connector.getBlacklist() ) )
743 {
744 log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath,
745 remoteRepository.getRepository().getName() );
746 return null;
747 }
748
749
750 try
751 {
752 validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
753 }
754 catch ( PolicyViolationException e )
755 {
756 String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
757 if ( fileExists( resource ) )
758 {
759 log.debug( "{} : using already present local file.", emsg );
760 return resource;
761 }
762
763 log.debug( emsg );
764 return null;
765 }
766
767 File workingDirectory = createWorkingDirectory( repository );
768 File tmpResource = new File( workingDirectory, resource.getName() );
769 File tmpMd5 = new File( workingDirectory, resource.getName() + ".md5" );
770 File tmpSha1 = new File( workingDirectory, resource.getName() + ".sha1" );
771
772 try
773 {
774
775 transferResources( connector, remoteRepository, tmpMd5, tmpSha1, tmpResource, url, remotePath, resource,
776 workingDirectory, repository );
777
778
779 try
780 {
781 validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
782 }
783 catch ( PolicyViolationException e )
784 {
785 log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() );
786 executeConsumers = false;
787 if ( !fileExists( tmpResource ) )
788 {
789 resource = null;
790 }
791 }
792
793 if ( resource != null )
794 {
795 synchronized ( resource.getAbsolutePath().intern() )
796 {
797 File directory = resource.getParentFile();
798 moveFileIfExists( tmpMd5, directory );
799 moveFileIfExists( tmpSha1, directory );
800 moveFileIfExists( tmpResource, directory );
801 }
802 }
803 }
804 finally
805 {
806 FileUtils.deleteQuietly( workingDirectory );
807 }
808
809 if ( executeConsumers )
810 {
811
812
813 queueRepositoryTask( connector.getSourceRepository().getRepository().getId(), resource );
814 }
815
816 return resource;
817 }
818
819 private void queueRepositoryTask( String repositoryId, File localFile )
820 {
821 RepositoryTask task = new RepositoryTask();
822 task.setRepositoryId( repositoryId );
823 task.setResourceFile( localFile );
824 task.setUpdateRelatedArtifacts( true );
825 task.setScanAll( true );
826
827 try
828 {
829 scheduler.queueTask( task );
830 }
831 catch ( TaskQueueException e )
832 {
833 log.error( "Unable to queue repository task to execute consumers on resource file ['" + localFile.getName()
834 + "']." );
835 }
836 }
837
838
839
840
841
842
843
844 private void moveFileIfExists( File fileToMove, File directory )
845 throws ProxyException
846 {
847 if ( fileToMove != null && fileToMove.exists() )
848 {
849 File newLocation = new File( directory, fileToMove.getName() );
850 moveTempToTarget( fileToMove, newLocation );
851 }
852 }
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868 private void transferChecksum( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
869 ManagedRepositoryContent repository, File resource, File tmpDirectory, String ext,
870 File destFile )
871 throws ProxyException
872 {
873 String url = remoteRepository.getURL().getUrl() + remotePath + ext;
874
875
876 if ( urlFailureCache.hasFailedBefore( url ) )
877 {
878 return;
879 }
880
881 try
882 {
883 transferSimpleFile( wagon, remoteRepository, remotePath + ext, repository, resource, destFile );
884 log.debug( "Checksum {} Downloaded: {} to move to {}", url, destFile, resource );
885 }
886 catch ( NotFoundException e )
887 {
888 urlFailureCache.cacheFailure( url );
889 log.debug( "Transfer failed, checksum not found: {}", url );
890
891 }
892 catch ( NotModifiedException e )
893 {
894 log.debug( "Transfer skipped, checksum not modified: {}", url );
895
896 }
897 catch ( ProxyException e )
898 {
899 urlFailureCache.cacheFailure( url );
900 log.warn( "Transfer failed on checksum: {} : {}", url, e.getMessage(), e );
901
902 throw e;
903 }
904 }
905
906
907
908
909
910
911
912
913
914
915
916 private void transferSimpleFile( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
917 ManagedRepositoryContent repository, File origFile, File destFile )
918 throws ProxyException
919 {
920 assert ( remotePath != null );
921
922
923 try
924 {
925 boolean success = false;
926
927 if ( !origFile.exists() )
928 {
929 log.debug( "Retrieving {} from {}", remotePath, remoteRepository.getRepository().getName() );
930 wagon.get( addParameters( remotePath, remoteRepository.getRepository() ), destFile );
931 success = true;
932
933
934 log.debug( "Downloaded successfully." );
935 }
936 else
937 {
938 log.debug( "Retrieving {} from {} if updated", remotePath, remoteRepository.getRepository().getName() );
939 success = wagon.getIfNewer( addParameters( remotePath, remoteRepository.getRepository() ), destFile,
940 origFile.lastModified() );
941 if ( !success )
942 {
943 throw new NotModifiedException(
944 "Not downloaded, as local file is newer than remote side: " + origFile.getAbsolutePath() );
945 }
946
947 if ( destFile.exists() )
948 {
949 log.debug( "Downloaded successfully." );
950 }
951 }
952 }
953 catch ( ResourceDoesNotExistException e )
954 {
955 throw new NotFoundException(
956 "Resource [" + remoteRepository.getURL() + "/" + remotePath + "] does not exist: " + e.getMessage(),
957 e );
958 }
959 catch ( WagonException e )
960 {
961
962
963 String msg =
964 "Download failure on resource [" + remoteRepository.getURL() + "/" + remotePath + "]:" + e.getMessage();
965 if ( e.getCause() != null )
966 {
967 msg += " (cause: " + e.getCause() + ")";
968 }
969 throw new ProxyException( msg, e );
970 }
971 }
972
973
974
975
976
977
978
979
980
981
982
983
984 private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<String, String> settings,
985 Properties request, File localFile )
986 throws PolicyViolationException
987 {
988 for ( Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
989 {
990
991
992 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
993 DownloadPolicy policy = entry.getValue();
994 String defaultSetting = policy.getDefaultOption();
995
996 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
997
998 log.debug( "Applying [{}] policy with [{}]", key, setting );
999 try
1000 {
1001 policy.applyPolicy( setting, request, localFile );
1002 }
1003 catch ( PolicyConfigurationException e )
1004 {
1005 log.error( e.getMessage(), e );
1006 }
1007 }
1008 }
1009
1010 private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<String, String> settings,
1011 Properties request, ArtifactReference artifact, RemoteRepositoryContent content,
1012 File localFile, Exception exception, Map<String, Exception> previousExceptions )
1013 throws ProxyDownloadException
1014 {
1015 boolean process = true;
1016 for ( Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
1017 {
1018
1019
1020
1021 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
1022 DownloadErrorPolicy policy = entry.getValue();
1023 String defaultSetting = policy.getDefaultOption();
1024 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
1025
1026 log.debug( "Applying [{}] policy with [{}]", key, setting );
1027 try
1028 {
1029
1030 process = policy.applyPolicy( setting, request, localFile, exception, previousExceptions );
1031 if ( !process )
1032 {
1033 break;
1034 }
1035 }
1036 catch ( PolicyConfigurationException e )
1037 {
1038 log.error( e.getMessage(), e );
1039 }
1040 }
1041
1042 if ( process )
1043 {
1044
1045 if ( !previousExceptions.containsKey( content.getId() ) )
1046 {
1047 throw new ProxyDownloadException(
1048 "An error occurred in downloading from the remote repository, and the policy is to fail immediately",
1049 content.getId(), exception );
1050 }
1051 }
1052 else
1053 {
1054
1055 previousExceptions.remove( content.getId() );
1056 }
1057
1058 log.warn(
1059 "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}",
1060 content.getRepository().getId(), Keys.toKey( artifact ), exception.getMessage() );
1061 log.debug( "Full stack trace", exception );
1062 }
1063
1064
1065
1066
1067
1068
1069
1070 private File createWorkingDirectory( ManagedRepositoryContent repository )
1071 {
1072 try
1073 {
1074 return Files.createTempDirectory( "temp" ).toFile();
1075 }
1076 catch ( IOException e )
1077 {
1078 throw new RuntimeException( e.getMessage(), e );
1079 }
1080
1081 }
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091 private void moveTempToTarget( File temp, File target )
1092 throws ProxyException
1093 {
1094
1095 Lock lock;
1096 try
1097 {
1098 lock = fileLockManager.writeFileLock( target );
1099 if ( lock.getFile().exists() && !lock.getFile().delete() )
1100 {
1101 throw new ProxyException( "Unable to overwrite existing target file: " + target.getAbsolutePath() );
1102 }
1103
1104 lock.getFile().getParentFile().mkdirs();
1105
1106 if ( !temp.renameTo( lock.getFile() ) )
1107 {
1108 log.warn( "Unable to rename tmp file to its final name... resorting to copy command." );
1109
1110 try
1111 {
1112 FileUtils.copyFile( temp, lock.getFile() );
1113 }
1114 catch ( IOException e )
1115 {
1116 if ( lock.getFile().exists() )
1117 {
1118 log.debug( "Tried to copy file {} to {} but file with this name already exists.",
1119 temp.getName(), lock.getFile().getAbsolutePath() );
1120 }
1121 else
1122 {
1123 throw new ProxyException(
1124 "Cannot copy tmp file " + temp.getAbsolutePath() + " to its final location", e );
1125 }
1126 }
1127 finally
1128 {
1129 FileUtils.deleteQuietly( temp );
1130 }
1131 }
1132 }
1133 catch ( FileLockException | FileLockTimeoutException e )
1134 {
1135 throw new ProxyException( e.getMessage(), e );
1136 }
1137 }
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147 private boolean connectToRepository( ProxyConnector connector, Wagon wagon,
1148 RemoteRepositoryContent remoteRepository )
1149 {
1150 boolean connected = false;
1151
1152 final ProxyInfo networkProxy =
1153 connector.getProxyId() == null ? null : this.networkProxyMap.get( connector.getProxyId() );
1154
1155 if ( log.isDebugEnabled() )
1156 {
1157 if ( networkProxy != null )
1158 {
1159
1160 String msg = "Using network proxy " + networkProxy.getHost() + ":" + networkProxy.getPort()
1161 + " to connect to remote repository " + remoteRepository.getURL();
1162 if ( networkProxy.getNonProxyHosts() != null )
1163 {
1164 msg += "; excluding hosts: " + networkProxy.getNonProxyHosts();
1165 }
1166 if ( StringUtils.isNotBlank( networkProxy.getUserName() ) )
1167 {
1168 msg += "; as user: " + networkProxy.getUserName();
1169 }
1170 log.debug( msg );
1171 }
1172 }
1173
1174 AuthenticationInfo authInfo = null;
1175 String username = remoteRepository.getRepository().getUserName();
1176 String password = remoteRepository.getRepository().getPassword();
1177
1178 if ( StringUtils.isNotBlank( username ) && StringUtils.isNotBlank( password ) )
1179 {
1180 log.debug( "Using username {} to connect to remote repository {}", username, remoteRepository.getURL() );
1181 authInfo = new AuthenticationInfo();
1182 authInfo.setUserName( username );
1183 authInfo.setPassword( password );
1184 }
1185
1186
1187 long timeoutInMilliseconds = TimeUnit.MILLISECONDS.convert( remoteRepository.getRepository().getTimeout(),
1188 TimeUnit.SECONDS );
1189
1190
1191
1192 wagon.setReadTimeout( (int) timeoutInMilliseconds );
1193 wagon.setTimeout( (int) timeoutInMilliseconds );
1194
1195 try
1196 {
1197 Repository wagonRepository =
1198 new Repository( remoteRepository.getId(), remoteRepository.getURL().toString() );
1199 wagon.connect( wagonRepository, authInfo, networkProxy );
1200 connected = true;
1201 }
1202 catch ( ConnectionException | AuthenticationException e )
1203 {
1204 log.warn( "Could not connect to {}: {}", remoteRepository.getRepository().getName(), e.getMessage() );
1205 connected = false;
1206 }
1207
1208 return connected;
1209 }
1210
1211
1212
1213
1214
1215
1216
1217
1218 private boolean matchesPattern( String path, List<String> patterns )
1219 {
1220 if ( CollectionUtils.isEmpty( patterns ) )
1221 {
1222 return false;
1223 }
1224
1225 if ( !path.startsWith( "/" ) )
1226 {
1227 path = "/" + path;
1228 }
1229
1230 for ( String pattern : patterns )
1231 {
1232 if ( !pattern.startsWith( "/" ) )
1233 {
1234 pattern = "/" + pattern;
1235 }
1236
1237 if ( SelectorUtils.matchPath( pattern, path, false ) )
1238 {
1239 return true;
1240 }
1241 }
1242
1243 return false;
1244 }
1245
1246
1247
1248
1249 @Override
1250 public List<ProxyConnector> getProxyConnectors( ManagedRepositoryContent repository )
1251 {
1252
1253 if ( !this.proxyConnectorMap.containsKey( repository.getId() ) )
1254 {
1255 return Collections.emptyList();
1256 }
1257 List<ProxyConnector> ret = new ArrayList<>( this.proxyConnectorMap.get( repository.getId() ) );
1258
1259 Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
1260 return ret;
1261
1262 }
1263
1264 @Override
1265 public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1266 {
1267 if ( ConfigurationNames.isNetworkProxy( propertyName )
1268 || ConfigurationNames.isManagedRepositories( propertyName )
1269 || ConfigurationNames.isRemoteRepositories( propertyName )
1270 || ConfigurationNames.isProxyConnector( propertyName ) )
1271 {
1272 initConnectorsAndNetworkProxies();
1273 }
1274 }
1275
1276 protected String addParameters( String path, RemoteRepository remoteRepository )
1277 {
1278 if ( remoteRepository.getExtraParameters().isEmpty() )
1279 {
1280 return path;
1281 }
1282
1283 boolean question = false;
1284
1285 StringBuilder res = new StringBuilder( path == null ? "" : path );
1286
1287 for ( Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
1288 {
1289 if ( !question )
1290 {
1291 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
1292 }
1293 }
1294
1295 return res.toString();
1296 }
1297
1298
1299 @Override
1300 public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1301 {
1302
1303 }
1304
1305 public ArchivaConfiguration getArchivaConfiguration()
1306 {
1307 return archivaConfiguration;
1308 }
1309
1310 public void setArchivaConfiguration( ArchivaConfiguration archivaConfiguration )
1311 {
1312 this.archivaConfiguration = archivaConfiguration;
1313 }
1314
1315 public RepositoryContentFactory getRepositoryFactory()
1316 {
1317 return repositoryFactory;
1318 }
1319
1320 public void setRepositoryFactory( RepositoryContentFactory repositoryFactory )
1321 {
1322 this.repositoryFactory = repositoryFactory;
1323 }
1324
1325 public MetadataTools getMetadataTools()
1326 {
1327 return metadataTools;
1328 }
1329
1330 public void setMetadataTools( MetadataTools metadataTools )
1331 {
1332 this.metadataTools = metadataTools;
1333 }
1334
1335 public UrlFailureCache getUrlFailureCache()
1336 {
1337 return urlFailureCache;
1338 }
1339
1340 public void setUrlFailureCache( UrlFailureCache urlFailureCache )
1341 {
1342 this.urlFailureCache = urlFailureCache;
1343 }
1344
1345 public WagonFactory getWagonFactory()
1346 {
1347 return wagonFactory;
1348 }
1349
1350 public void setWagonFactory( WagonFactory wagonFactory )
1351 {
1352 this.wagonFactory = wagonFactory;
1353 }
1354
1355 public Map<String, PreDownloadPolicy> getPreDownloadPolicies()
1356 {
1357 return preDownloadPolicies;
1358 }
1359
1360 public void setPreDownloadPolicies( Map<String, PreDownloadPolicy> preDownloadPolicies )
1361 {
1362 this.preDownloadPolicies = preDownloadPolicies;
1363 }
1364
1365 public Map<String, PostDownloadPolicy> getPostDownloadPolicies()
1366 {
1367 return postDownloadPolicies;
1368 }
1369
1370 public void setPostDownloadPolicies( Map<String, PostDownloadPolicy> postDownloadPolicies )
1371 {
1372 this.postDownloadPolicies = postDownloadPolicies;
1373 }
1374
1375 public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies()
1376 {
1377 return downloadErrorPolicies;
1378 }
1379
1380 public void setDownloadErrorPolicies( Map<String, DownloadErrorPolicy> downloadErrorPolicies )
1381 {
1382 this.downloadErrorPolicies = downloadErrorPolicies;
1383 }
1384 }