This project has retired. For details please refer to its
Attic page.
1 package org.apache.archiva.proxy;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import org.apache.archiva.admin.model.RepositoryAdminException;
23 import org.apache.archiva.admin.model.beans.NetworkProxy;
24 import org.apache.archiva.admin.model.beans.ProxyConnectorRuleType;
25 import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
26 import org.apache.archiva.common.filelock.FileLockException;
27 import org.apache.archiva.common.filelock.FileLockManager;
28 import org.apache.archiva.common.filelock.FileLockTimeoutException;
29 import org.apache.archiva.common.filelock.Lock;
30 import org.apache.archiva.configuration.ArchivaConfiguration;
31 import org.apache.archiva.configuration.Configuration;
32 import org.apache.archiva.configuration.ConfigurationNames;
33 import org.apache.archiva.configuration.NetworkProxyConfiguration;
34 import org.apache.archiva.configuration.ProxyConnectorConfiguration;
35 import org.apache.archiva.configuration.ProxyConnectorRuleConfiguration;
36 import org.apache.archiva.model.ArtifactReference;
37 import org.apache.archiva.model.Keys;
38 import org.apache.archiva.model.RepositoryURL;
39 import org.apache.archiva.policies.DownloadErrorPolicy;
40 import org.apache.archiva.policies.DownloadPolicy;
41 import org.apache.archiva.policies.PolicyConfigurationException;
42 import org.apache.archiva.policies.PolicyViolationException;
43 import org.apache.archiva.policies.PostDownloadPolicy;
44 import org.apache.archiva.policies.PreDownloadPolicy;
45 import org.apache.archiva.policies.ProxyDownloadException;
46 import org.apache.archiva.policies.urlcache.UrlFailureCache;
47 import org.apache.archiva.proxy.common.WagonFactory;
48 import org.apache.archiva.proxy.common.WagonFactoryException;
49 import org.apache.archiva.proxy.common.WagonFactoryRequest;
50 import org.apache.archiva.proxy.model.ProxyConnector;
51 import org.apache.archiva.proxy.model.ProxyFetchResult;
52 import org.apache.archiva.proxy.model.RepositoryProxyConnectors;
53 import org.apache.archiva.redback.components.registry.Registry;
54 import org.apache.archiva.redback.components.registry.RegistryListener;
55 import org.apache.archiva.redback.components.taskqueue.TaskQueueException;
56 import org.apache.archiva.repository.ManagedRepository;
57 import org.apache.archiva.repository.ManagedRepositoryContent;
58 import org.apache.archiva.repository.PasswordCredentials;
59 import org.apache.archiva.repository.RemoteRepository;
60 import org.apache.archiva.repository.RemoteRepositoryContent;
61 import org.apache.archiva.repository.RepositoryContentFactory;
62 import org.apache.archiva.repository.RepositoryCredentials;
63 import org.apache.archiva.repository.RepositoryRegistry;
64 import org.apache.archiva.repository.metadata.MetadataTools;
65 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
66 import org.apache.archiva.scheduler.ArchivaTaskScheduler;
67 import org.apache.archiva.scheduler.repository.model.RepositoryTask;
68 import org.apache.commons.collections4.CollectionUtils;
69 import org.apache.commons.io.FilenameUtils;
70 import org.apache.commons.lang.StringUtils;
71 import org.apache.commons.lang.SystemUtils;
72 import org.apache.maven.wagon.ConnectionException;
73 import org.apache.maven.wagon.ResourceDoesNotExistException;
74 import org.apache.maven.wagon.Wagon;
75 import org.apache.maven.wagon.WagonException;
76 import org.apache.maven.wagon.authentication.AuthenticationException;
77 import org.apache.maven.wagon.authentication.AuthenticationInfo;
78 import org.apache.maven.wagon.proxy.ProxyInfo;
79 import org.apache.maven.wagon.repository.Repository;
80 import org.apache.tools.ant.types.selectors.SelectorUtils;
81 import org.slf4j.Logger;
82 import org.slf4j.LoggerFactory;
83 import org.slf4j.MarkerFactory;
84 import org.springframework.stereotype.Service;
85
86 import javax.annotation.PostConstruct;
87 import javax.inject.Inject;
88 import javax.inject.Named;
89 import java.io.IOException;
90 import java.nio.file.Files;
91 import java.nio.file.Path;
92 import java.nio.file.Paths;
93 import java.util.ArrayList;
94 import java.util.Collections;
95 import java.util.LinkedHashMap;
96 import java.util.List;
97 import java.util.Map;
98 import java.util.Map.Entry;
99 import java.util.Properties;
100 import java.util.concurrent.ConcurrentHashMap;
101 import java.util.concurrent.ConcurrentMap;
102
103
104
105
106
107
108 @Service("repositoryProxyConnectors#default")
109 public class DefaultRepositoryProxyConnectors
110 implements RepositoryProxyConnectors, RegistryListener
111 {
112 private Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyConnectors.class );
113
114 @Inject
115 @Named(value = "archivaConfiguration#default")
116 private ArchivaConfiguration archivaConfiguration;
117
118 @Inject
119 @Named(value = "repositoryContentFactory#default")
120 private RepositoryContentFactory repositoryFactory;
121
122 @Inject
123 @Named(value = "metadataTools#default")
124 private MetadataTools metadataTools;
125
126 @Inject
127 private Map<String, PreDownloadPolicy> preDownloadPolicies;
128
129 @Inject
130 private Map<String, PostDownloadPolicy> postDownloadPolicies;
131
132 @Inject
133 private Map<String, DownloadErrorPolicy> downloadErrorPolicies;
134
135 @Inject
136 private UrlFailureCache urlFailureCache;
137
138 private ConcurrentMap<String, List<ProxyConnector>> proxyConnectorMap = new ConcurrentHashMap<>();
139
140 private ConcurrentMap<String, ProxyInfo> networkProxyMap = new ConcurrentHashMap<>();
141
142 @Inject
143 private WagonFactory wagonFactory;
144
145 @Inject
146 @Named(value = "archivaTaskScheduler#repository")
147 private ArchivaTaskScheduler<RepositoryTask> scheduler;
148
149 @Inject
150 private RepositoryRegistry repositoryRegistry;
151
152 @Inject
153 private NetworkProxyAdmin networkProxyAdmin;
154
155 @Inject
156 @Named(value = "fileLockManager#default")
157 private FileLockManager fileLockManager;
158
159 @PostConstruct
160 public void initialize()
161 {
162 initConnectorsAndNetworkProxies();
163 archivaConfiguration.addChangeListener( this );
164
165 }
166
167 @SuppressWarnings("unchecked")
168 private void initConnectorsAndNetworkProxies()
169 {
170
171 ProxyConnectorOrderComparator proxyOrderSorter = new ProxyConnectorOrderComparator();
172 this.proxyConnectorMap.clear();
173
174 Configuration configuration = archivaConfiguration.getConfiguration();
175
176 List<ProxyConnectorRuleConfiguration> allProxyConnectorRuleConfigurations =
177 configuration.getProxyConnectorRuleConfigurations();
178
179 List<ProxyConnectorConfiguration> proxyConfigs = configuration.getProxyConnectors();
180 for ( ProxyConnectorConfiguration proxyConfig : proxyConfigs )
181 {
182 String key = proxyConfig.getSourceRepoId();
183
184
185 ProxyConnector connector = new ProxyConnector();
186
187 ManagedRepository repo = repositoryRegistry.getManagedRepository( proxyConfig.getSourceRepoId( ) );
188 if (repo==null) {
189 log.error("Cannot find source repository after config change "+proxyConfig.getSourceRepoId());
190 continue;
191 }
192 connector.setSourceRepository(repo.getContent());
193 RemoteRepository rRepo = repositoryRegistry.getRemoteRepository( proxyConfig.getTargetRepoId() );
194 if (rRepo==null) {
195 log.error("Cannot find target repository after config change "+proxyConfig.getSourceRepoId());
196 continue;
197 }
198 connector.setTargetRepository(rRepo.getContent());
199
200 connector.setProxyId( proxyConfig.getProxyId() );
201 connector.setPolicies( proxyConfig.getPolicies() );
202 connector.setOrder( proxyConfig.getOrder() );
203 connector.setDisabled( proxyConfig.isDisabled() );
204
205
206 List<String> blacklist = new ArrayList<>( 0 );
207 if ( CollectionUtils.isNotEmpty( proxyConfig.getBlackListPatterns() ) )
208 {
209 blacklist.addAll( proxyConfig.getBlackListPatterns() );
210 }
211 connector.setBlacklist( blacklist );
212
213
214 List<String> whitelist = new ArrayList<>( 0 );
215 if ( CollectionUtils.isNotEmpty( proxyConfig.getWhiteListPatterns() ) )
216 {
217 whitelist.addAll( proxyConfig.getWhiteListPatterns() );
218 }
219 connector.setWhitelist( whitelist );
220
221 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations =
222 findProxyConnectorRules( connector.getSourceRepository().getId(),
223 connector.getTargetRepository().getId(),
224 allProxyConnectorRuleConfigurations );
225
226 if ( !proxyConnectorRuleConfigurations.isEmpty() )
227 {
228 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : proxyConnectorRuleConfigurations )
229 {
230 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
231 ProxyConnectorRuleType.BLACK_LIST.getRuleType() ) )
232 {
233 connector.getBlacklist().add( proxyConnectorRuleConfiguration.getPattern() );
234 }
235
236 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
237 ProxyConnectorRuleType.WHITE_LIST.getRuleType() ) )
238 {
239 connector.getWhitelist().add( proxyConnectorRuleConfiguration.getPattern() );
240 }
241 }
242 }
243
244
245 List<ProxyConnector> connectors = this.proxyConnectorMap.get( key );
246 if ( connectors == null )
247 {
248
249 connectors = new ArrayList<>( 1 );
250 }
251
252
253 connectors.add( connector );
254
255
256 Collections.sort( connectors, proxyOrderSorter );
257
258
259 this.proxyConnectorMap.put( key, connectors );
260
261
262 }
263
264 this.networkProxyMap.clear();
265
266 List<NetworkProxyConfiguration> networkProxies = archivaConfiguration.getConfiguration().getNetworkProxies();
267 for ( NetworkProxyConfiguration networkProxyConfig : networkProxies )
268 {
269 String key = networkProxyConfig.getId();
270
271 ProxyInfo proxy = new ProxyInfo();
272
273 proxy.setType( networkProxyConfig.getProtocol() );
274 proxy.setHost( networkProxyConfig.getHost() );
275 proxy.setPort( networkProxyConfig.getPort() );
276 proxy.setUserName( networkProxyConfig.getUsername() );
277 proxy.setPassword( networkProxyConfig.getPassword() );
278
279 this.networkProxyMap.put( key, proxy );
280 }
281
282 }
283
284 private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules( String sourceRepository,
285 String targetRepository,
286 List<ProxyConnectorRuleConfiguration> all )
287 {
288 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = new ArrayList<>();
289
290 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all )
291 {
292 for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() )
293 {
294 if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals(
295 targetRepository, proxyConnector.getTargetRepoId() ) )
296 {
297 proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration );
298 }
299 }
300 }
301
302 return proxyConnectorRuleConfigurations;
303 }
304
305 @Override
306 public Path fetchFromProxies( ManagedRepositoryContent repository, ArtifactReference artifact )
307 throws ProxyDownloadException
308 {
309 Path localFile = toLocalFile( repository, artifact );
310
311 Properties requestProperties = new Properties();
312 requestProperties.setProperty( "filetype", "artifact" );
313 requestProperties.setProperty( "version", artifact.getVersion() );
314 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
315
316 List<ProxyConnector> connectors = getProxyConnectors( repository );
317 Map<String, Exception> previousExceptions = new LinkedHashMap<>();
318 for ( ProxyConnector connector : connectors )
319 {
320 if ( connector.isDisabled() )
321 {
322 continue;
323 }
324
325 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
326 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
327
328 String targetPath = targetRepository.toPath( artifact );
329
330 if ( SystemUtils.IS_OS_WINDOWS )
331 {
332
333 targetPath = FilenameUtils.separatorsToUnix( targetPath );
334 }
335
336 try
337 {
338 Path downloadedFile =
339 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
340 true );
341
342 if ( fileExists( downloadedFile ) )
343 {
344 log.debug( "Successfully transferred: {}", downloadedFile.toAbsolutePath() );
345 return downloadedFile;
346 }
347 }
348 catch ( NotFoundException e )
349 {
350 log.debug( "Artifact {} not found on repository \"{}\".", Keys.toKey( artifact ),
351 targetRepository.getRepository().getId() );
352 }
353 catch ( NotModifiedException e )
354 {
355 log.debug( "Artifact {} not updated on repository \"{}\".", Keys.toKey( artifact ),
356 targetRepository.getRepository().getId() );
357 }
358 catch ( ProxyException | RepositoryAdminException e )
359 {
360 validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
361 targetRepository, localFile, e, previousExceptions );
362 }
363 }
364
365 if ( !previousExceptions.isEmpty() )
366 {
367 throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
368 previousExceptions );
369 }
370
371 log.debug( "Exhausted all target repositories, artifact {} not found.", Keys.toKey( artifact ) );
372
373 return null;
374 }
375
376 @Override
377 public Path fetchFromProxies( ManagedRepositoryContent repository, String path )
378 {
379 Path localFile = Paths.get( repository.getRepoRoot(), path );
380
381
382 if ( Files.exists(localFile) )
383 {
384 return null;
385 }
386
387 Properties requestProperties = new Properties();
388 requestProperties.setProperty( "filetype", "resource" );
389 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
390
391 List<ProxyConnector> connectors = getProxyConnectors( repository );
392 for ( ProxyConnector connector : connectors )
393 {
394 if ( connector.isDisabled() )
395 {
396 continue;
397 }
398
399 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
400 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
401
402 String targetPath = path;
403
404 try
405 {
406 Path downloadedFile =
407 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
408 false );
409
410 if ( fileExists( downloadedFile ) )
411 {
412 log.debug( "Successfully transferred: {}", downloadedFile.toAbsolutePath() );
413 return downloadedFile;
414 }
415 }
416 catch ( NotFoundException e )
417 {
418 log.debug( "Resource {} not found on repository \"{}\".", path,
419 targetRepository.getRepository().getId() );
420 }
421 catch ( NotModifiedException e )
422 {
423 log.debug( "Resource {} not updated on repository \"{}\".", path,
424 targetRepository.getRepository().getId() );
425 }
426 catch ( ProxyException e )
427 {
428 log.warn(
429 "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
430 targetRepository.getRepository().getId(), path, e.getMessage() );
431 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
432 "Transfer error from repository \"{}"
433 + "\" for resource {}, continuing to next repository. Error message: {}",
434 targetRepository.getRepository().getId(), path, e.getMessage(), e );
435 }
436 catch ( RepositoryAdminException e )
437 {
438 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
439 "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
440 targetRepository.getRepository().getId(), path, e.getMessage(), e );
441 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ), "Full stack trace", e );
442 }
443 }
444
445 log.debug( "Exhausted all target repositories, resource {} not found.", path );
446
447 return null;
448 }
449
450 @Override
451 public ProxyFetchResult fetchMetadataFromProxies( ManagedRepositoryContent repository, String logicalPath )
452 {
453 Path localFile = Paths.get( repository.getRepoRoot(), logicalPath );
454
455 Properties requestProperties = new Properties();
456 requestProperties.setProperty( "filetype", "metadata" );
457 boolean metadataNeedsUpdating = false;
458 long originalTimestamp = getLastModified( localFile );
459
460 List<ProxyConnector> connectors = new ArrayList<>( getProxyConnectors( repository ) );
461 for ( ProxyConnector connector : connectors )
462 {
463 if ( connector.isDisabled() )
464 {
465 continue;
466 }
467
468 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
469
470 Path localRepoFile = toLocalRepoFile( repository, targetRepository, logicalPath );
471 long originalMetadataTimestamp = getLastModified( localRepoFile );
472
473 try
474 {
475 transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties,
476 true );
477
478 if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
479 {
480 metadataNeedsUpdating = true;
481 }
482 }
483 catch ( NotFoundException e )
484 {
485
486 log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath,
487 targetRepository.getRepository().getId(), e );
488
489 }
490 catch ( NotModifiedException e )
491 {
492
493 log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath,
494 targetRepository.getRepository().getId(), e );
495
496 }
497 catch ( ProxyException | RepositoryAdminException e )
498 {
499 log.warn(
500 "Transfer error from repository {} for versioned Metadata {}, continuing to next repository. Error message: {}",
501 targetRepository.getRepository().getId(), logicalPath, e.getMessage() );
502 log.debug( "Full stack trace", e );
503 }
504 }
505
506 if ( hasBeenUpdated( localFile, originalTimestamp ) )
507 {
508 metadataNeedsUpdating = true;
509 }
510
511 if ( metadataNeedsUpdating || !Files.exists(localFile))
512 {
513 try
514 {
515 metadataTools.updateMetadata( repository, logicalPath );
516 }
517 catch ( RepositoryMetadataException e )
518 {
519 log.warn( "Unable to update metadata {}:{}", localFile.toAbsolutePath(), e.getMessage(), e );
520 }
521
522 }
523
524 if ( fileExists( localFile ) )
525 {
526 return new ProxyFetchResult( localFile, metadataNeedsUpdating );
527 }
528
529 return new ProxyFetchResult( null, false );
530 }
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547 protected void transferResources( ProxyConnector connector, RemoteRepositoryContent remoteRepository, Path tmpMd5,
548 Path tmpSha1, Path tmpResource, String url, String remotePath, Path resource,
549 Path workingDirectory, ManagedRepositoryContent repository )
550 throws ProxyException, NotModifiedException, RepositoryAdminException
551 {
552 Wagon wagon = null;
553 try
554 {
555 RepositoryURL repoUrl = remoteRepository.getURL();
556 String protocol = repoUrl.getProtocol();
557 NetworkProxy networkProxy = null;
558 if ( StringUtils.isNotBlank( connector.getProxyId() ) )
559 {
560 networkProxy = networkProxyAdmin.getNetworkProxy( connector.getProxyId() );
561 }
562 WagonFactoryRequest wagonFactoryRequest = new WagonFactoryRequest( "wagon#" + protocol,
563 remoteRepository.getRepository().getExtraHeaders() ).networkProxy(
564 networkProxy );
565 wagon = wagonFactory.getWagon( wagonFactoryRequest );
566 if ( wagon == null )
567 {
568 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
569 }
570
571 if ( wagon == null )
572 {
573 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
574 }
575
576 boolean connected = connectToRepository( connector, wagon, remoteRepository );
577 if ( connected )
578 {
579 transferArtifact( wagon, remoteRepository, remotePath, repository, resource, workingDirectory,
580 tmpResource );
581
582
583
584
585 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".sha1",
586 tmpSha1 );
587 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".md5",
588 tmpMd5 );
589 }
590 }
591 catch ( NotFoundException e )
592 {
593 urlFailureCache.cacheFailure( url );
594 throw e;
595 }
596 catch ( NotModifiedException e )
597 {
598
599 throw e;
600 }
601 catch ( ProxyException e )
602 {
603 urlFailureCache.cacheFailure( url );
604 throw e;
605 }
606 catch ( WagonFactoryException e )
607 {
608 throw new ProxyException( e.getMessage(), e );
609 }
610 finally
611 {
612 if ( wagon != null )
613 {
614 try
615 {
616 wagon.disconnect();
617 }
618 catch ( ConnectionException e )
619 {
620 log.warn( "Unable to disconnect wagon.", e );
621 }
622 }
623 }
624 }
625
626 private void transferArtifact( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
627 ManagedRepositoryContent repository, Path resource, Path tmpDirectory,
628 Path destFile )
629 throws ProxyException
630 {
631 transferSimpleFile( wagon, remoteRepository, remotePath, repository, resource, destFile );
632 }
633
634 private long getLastModified( Path file )
635 {
636 if ( !Files.exists(file) || !Files.isRegularFile(file) )
637 {
638 return 0;
639 }
640
641 try
642 {
643 return Files.getLastModifiedTime(file).toMillis();
644 }
645 catch ( IOException e )
646 {
647 log.error("Could get the modified time of file {}", file.toAbsolutePath());
648 return 0;
649 }
650 }
651
652 private boolean hasBeenUpdated( Path file, long originalLastModified )
653 {
654 if ( !Files.exists(file) || !Files.isRegularFile(file) )
655 {
656 return false;
657 }
658
659 long currentLastModified = getLastModified( file );
660 return ( currentLastModified > originalLastModified );
661 }
662
663 private Path toLocalRepoFile( ManagedRepositoryContent repository, RemoteRepositoryContent targetRepository,
664 String targetPath )
665 {
666 String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
667 return Paths.get( repository.getRepoRoot(), repoPath );
668 }
669
670
671
672
673 @Override
674 public boolean hasProxies( ManagedRepositoryContent repository )
675 {
676 synchronized ( this.proxyConnectorMap )
677 {
678 return this.proxyConnectorMap.containsKey( repository.getId() );
679 }
680 }
681
682 private Path toLocalFile( ManagedRepositoryContent repository, ArtifactReference artifact )
683 {
684 return repository.toFile( artifact );
685 }
686
687
688
689
690
691
692
693 private boolean fileExists( Path file )
694 {
695 if ( file == null )
696 {
697 return false;
698 }
699
700 if ( !Files.exists(file))
701 {
702 return false;
703 }
704
705 return Files.isRegularFile(file);
706 }
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724 private Path transferFile( ProxyConnector connector, RemoteRepositoryContent remoteRepository, String remotePath,
725 ManagedRepositoryContent repository, Path resource, Properties requestProperties,
726 boolean executeConsumers )
727 throws ProxyException, NotModifiedException, RepositoryAdminException
728 {
729 String url = remoteRepository.getURL().getUrl();
730 if ( !url.endsWith( "/" ) )
731 {
732 url = url + "/";
733 }
734 url = url + remotePath;
735 requestProperties.setProperty( "url", url );
736
737
738 if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
739 {
740
741 if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
742 {
743 log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).",
744 remotePath, remoteRepository.getRepository().getName() );
745 return null;
746 }
747 }
748
749
750 if ( matchesPattern( remotePath, connector.getBlacklist() ) )
751 {
752 log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath,
753 remoteRepository.getRepository().getName() );
754 return null;
755 }
756
757
758 try
759 {
760 validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
761 }
762 catch ( PolicyViolationException e )
763 {
764 String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
765 if ( fileExists( resource ) )
766 {
767 log.debug( "{} : using already present local file.", emsg );
768 return resource;
769 }
770
771 log.debug( emsg );
772 return null;
773 }
774
775 Path workingDirectory = createWorkingDirectory( repository );
776 Path tmpResource = workingDirectory.resolve(resource.getFileName());
777 Path tmpMd5 = workingDirectory.resolve(resource.getFileName().toString() + ".md5" );
778 Path tmpSha1 = workingDirectory.resolve( resource.getFileName().toString() + ".sha1" );
779
780 try
781 {
782
783 transferResources( connector, remoteRepository, tmpMd5, tmpSha1, tmpResource, url, remotePath, resource,
784 workingDirectory, repository );
785
786
787 try
788 {
789 validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
790 }
791 catch ( PolicyViolationException e )
792 {
793 log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() );
794 executeConsumers = false;
795 if ( !fileExists( tmpResource ) )
796 {
797 resource = null;
798 }
799 }
800
801 if ( resource != null )
802 {
803 synchronized ( resource.toAbsolutePath().toString().intern() )
804 {
805 Path directory = resource.getParent();
806 moveFileIfExists( tmpMd5, directory );
807 moveFileIfExists( tmpSha1, directory );
808 moveFileIfExists( tmpResource, directory );
809 }
810 }
811 }
812 finally
813 {
814 org.apache.archiva.common.utils.FileUtils.deleteQuietly( workingDirectory );
815 }
816
817 if ( executeConsumers )
818 {
819
820
821 queueRepositoryTask( connector.getSourceRepository().getRepository().getId(), resource );
822 }
823
824 return resource;
825 }
826
827 private void queueRepositoryTask( String repositoryId, Path localFile )
828 {
829 RepositoryTask task = new RepositoryTask();
830 task.setRepositoryId( repositoryId );
831 task.setResourceFile( localFile );
832 task.setUpdateRelatedArtifacts( true );
833 task.setScanAll( true );
834
835 try
836 {
837 scheduler.queueTask( task );
838 }
839 catch ( TaskQueueException e )
840 {
841 log.error( "Unable to queue repository task to execute consumers on resource file ['{}"
842 + "'].", localFile.getFileName() );
843 }
844 }
845
846
847
848
849
850
851
852 private void moveFileIfExists( Path fileToMove, Path directory )
853 throws ProxyException
854 {
855 if ( fileToMove != null && Files.exists(fileToMove) )
856 {
857 Path newLocation = directory.resolve(fileToMove.getFileName());
858 moveTempToTarget( fileToMove, newLocation );
859 }
860 }
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876 private void transferChecksum( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
877 ManagedRepositoryContent repository, Path resource, Path tmpDirectory, String ext,
878 Path destFile )
879 throws ProxyException
880 {
881 String url = remoteRepository.getURL().getUrl() + remotePath + ext;
882
883
884 if ( urlFailureCache.hasFailedBefore( url ) )
885 {
886 return;
887 }
888
889 try
890 {
891 transferSimpleFile( wagon, remoteRepository, remotePath + ext, repository, resource, destFile );
892 log.debug( "Checksum {} Downloaded: {} to move to {}", url, destFile, resource );
893 }
894 catch ( NotFoundException e )
895 {
896 urlFailureCache.cacheFailure( url );
897 log.debug( "Transfer failed, checksum not found: {}", url );
898
899 }
900 catch ( NotModifiedException e )
901 {
902 log.debug( "Transfer skipped, checksum not modified: {}", url );
903
904 }
905 catch ( ProxyException e )
906 {
907 urlFailureCache.cacheFailure( url );
908 log.warn( "Transfer failed on checksum: {} : {}", url, e.getMessage(), e );
909
910 throw e;
911 }
912 }
913
914
915
916
917
918
919
920
921
922
923
924 private void transferSimpleFile( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
925 ManagedRepositoryContent repository, Path origFile, Path destFile )
926 throws ProxyException
927 {
928 assert ( remotePath != null );
929
930
931 try
932 {
933 boolean success = false;
934
935 if ( !Files.exists(origFile))
936 {
937 log.debug( "Retrieving {} from {}", remotePath, remoteRepository.getRepository().getName() );
938 wagon.get( addParameters( remotePath, remoteRepository.getRepository() ), destFile.toFile() );
939 success = true;
940
941
942 log.debug( "Downloaded successfully." );
943 }
944 else
945 {
946 log.debug( "Retrieving {} from {} if updated", remotePath, remoteRepository.getRepository().getName() );
947 try
948 {
949 success = wagon.getIfNewer( addParameters( remotePath, remoteRepository.getRepository() ), destFile.toFile(),
950 Files.getLastModifiedTime(origFile).toMillis());
951 }
952 catch ( IOException e )
953 {
954 throw new ProxyException( "Failed to the modification time of "+origFile.toAbsolutePath() );
955 }
956 if ( !success )
957 {
958 throw new NotModifiedException(
959 "Not downloaded, as local file is newer than remote side: " + origFile.toAbsolutePath() );
960 }
961
962 if ( Files.exists(destFile))
963 {
964 log.debug( "Downloaded successfully." );
965 }
966 }
967 }
968 catch ( ResourceDoesNotExistException e )
969 {
970 throw new NotFoundException(
971 "Resource [" + remoteRepository.getURL() + "/" + remotePath + "] does not exist: " + e.getMessage(),
972 e );
973 }
974 catch ( WagonException e )
975 {
976
977
978 String msg =
979 "Download failure on resource [" + remoteRepository.getURL() + "/" + remotePath + "]:" + e.getMessage();
980 if ( e.getCause() != null )
981 {
982 msg += " (cause: " + e.getCause() + ")";
983 }
984 throw new ProxyException( msg, e );
985 }
986 }
987
988
989
990
991
992
993
994
995
996
997
998
999 private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<String, String> settings,
1000 Properties request, Path localFile )
1001 throws PolicyViolationException
1002 {
1003 for ( Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
1004 {
1005
1006
1007 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
1008 DownloadPolicy policy = entry.getValue();
1009 String defaultSetting = policy.getDefaultOption();
1010
1011 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
1012
1013 log.debug( "Applying [{}] policy with [{}]", key, setting );
1014 try
1015 {
1016 policy.applyPolicy( setting, request, localFile );
1017 }
1018 catch ( PolicyConfigurationException e )
1019 {
1020 log.error( e.getMessage(), e );
1021 }
1022 }
1023 }
1024
1025 private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<String, String> settings,
1026 Properties request, ArtifactReference artifact, RemoteRepositoryContent content,
1027 Path localFile, Exception exception, Map<String, Exception> previousExceptions )
1028 throws ProxyDownloadException
1029 {
1030 boolean process = true;
1031 for ( Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
1032 {
1033
1034
1035
1036 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
1037 DownloadErrorPolicy policy = entry.getValue();
1038 String defaultSetting = policy.getDefaultOption();
1039 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
1040
1041 log.debug( "Applying [{}] policy with [{}]", key, setting );
1042 try
1043 {
1044
1045 process = policy.applyPolicy( setting, request, localFile, exception, previousExceptions );
1046 if ( !process )
1047 {
1048 break;
1049 }
1050 }
1051 catch ( PolicyConfigurationException e )
1052 {
1053 log.error( e.getMessage(), e );
1054 }
1055 }
1056
1057 if ( process )
1058 {
1059
1060 if ( !previousExceptions.containsKey( content.getId() ) )
1061 {
1062 throw new ProxyDownloadException(
1063 "An error occurred in downloading from the remote repository, and the policy is to fail immediately",
1064 content.getId(), exception );
1065 }
1066 }
1067 else
1068 {
1069
1070 previousExceptions.remove( content.getId() );
1071 }
1072
1073 log.warn(
1074 "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}",
1075 content.getRepository().getId(), Keys.toKey( artifact ), exception.getMessage() );
1076 log.debug( "Full stack trace", exception );
1077 }
1078
1079
1080
1081
1082
1083
1084
1085 private Path createWorkingDirectory( ManagedRepositoryContent repository )
1086 {
1087 try
1088 {
1089 return Files.createTempDirectory( "temp" );
1090 }
1091 catch ( IOException e )
1092 {
1093 throw new RuntimeException( e.getMessage(), e );
1094 }
1095
1096 }
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106 private void moveTempToTarget( Path temp, Path target )
1107 throws ProxyException
1108 {
1109
1110 Lock lock;
1111 try
1112 {
1113 lock = fileLockManager.writeFileLock( target );
1114 try {
1115 Files.deleteIfExists(lock.getFile());
1116 } catch (IOException e) {
1117 throw new ProxyException( "Unable to overwrite existing target file: " + target.toAbsolutePath() );
1118 }
1119
1120 try {
1121 Files.createDirectories(lock.getFile().getParent());
1122 } catch (IOException e) {
1123 throw new ProxyException("Unable to create parent directory "+lock.getFile().getParent());
1124 }
1125
1126 try
1127 {
1128 Files.move(temp, lock.getFile() );
1129 }
1130 catch ( IOException e )
1131 {
1132 log.warn( "Unable to rename tmp file to its final name... resorting to copy command." );
1133
1134 try
1135 {
1136 Files.copy( temp, lock.getFile());
1137 }
1138 catch ( IOException e2 )
1139 {
1140 if ( Files.exists(lock.getFile()) )
1141 {
1142 log.debug( "Tried to copy file {} to {} but file with this name already exists.",
1143 temp.getFileName(), lock.getFile().toAbsolutePath() );
1144 }
1145 else
1146 {
1147 throw new ProxyException(
1148 "Cannot copy tmp file " + temp.toAbsolutePath() + " to its final location", e2 );
1149 }
1150 }
1151 finally
1152 {
1153 org.apache.archiva.common.utils.FileUtils.deleteQuietly( temp );
1154 }
1155 }
1156
1157 }
1158 catch ( FileLockException | FileLockTimeoutException e )
1159 {
1160 throw new ProxyException( e.getMessage(), e );
1161 }
1162 }
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172 private boolean connectToRepository( ProxyConnector connector, Wagon wagon,
1173 RemoteRepositoryContent remoteRepository )
1174 {
1175 boolean connected = false;
1176
1177 final ProxyInfo networkProxy =
1178 connector.getProxyId() == null ? null : this.networkProxyMap.get( connector.getProxyId() );
1179
1180 if ( log.isDebugEnabled() )
1181 {
1182 if ( networkProxy != null )
1183 {
1184
1185 String msg = "Using network proxy " + networkProxy.getHost() + ":" + networkProxy.getPort()
1186 + " to connect to remote repository " + remoteRepository.getURL();
1187 if ( networkProxy.getNonProxyHosts() != null )
1188 {
1189 msg += "; excluding hosts: " + networkProxy.getNonProxyHosts();
1190 }
1191 if ( StringUtils.isNotBlank( networkProxy.getUserName() ) )
1192 {
1193 msg += "; as user: " + networkProxy.getUserName();
1194 }
1195 log.debug( msg );
1196 }
1197 }
1198
1199 AuthenticationInfo authInfo = null;
1200 String username = "";
1201 String password = "";
1202 RepositoryCredentials repCred = remoteRepository.getRepository().getLoginCredentials();
1203 if (repCred!=null && repCred instanceof PasswordCredentials) {
1204 PasswordCredentials pwdCred = (PasswordCredentials) repCred;
1205 username = pwdCred.getUsername();
1206 password = pwdCred.getPassword()==null ? "" : new String(pwdCred.getPassword());
1207 }
1208
1209 if ( StringUtils.isNotBlank( username ) && StringUtils.isNotBlank( password ) )
1210 {
1211 log.debug( "Using username {} to connect to remote repository {}", username, remoteRepository.getURL() );
1212 authInfo = new AuthenticationInfo();
1213 authInfo.setUserName( username );
1214 authInfo.setPassword( password );
1215 }
1216
1217
1218
1219 long timeoutInMilliseconds = remoteRepository.getRepository().getTimeout().toMillis();
1220
1221
1222
1223 wagon.setReadTimeout( (int) timeoutInMilliseconds );
1224 wagon.setTimeout( (int) timeoutInMilliseconds );
1225
1226 try
1227 {
1228 Repository wagonRepository =
1229 new Repository( remoteRepository.getId(), remoteRepository.getURL().toString() );
1230 wagon.connect( wagonRepository, authInfo, networkProxy );
1231 connected = true;
1232 }
1233 catch ( ConnectionException | AuthenticationException e )
1234 {
1235 log.warn( "Could not connect to {}: {}", remoteRepository.getRepository().getName(), e.getMessage() );
1236 connected = false;
1237 }
1238
1239 return connected;
1240 }
1241
1242
1243
1244
1245
1246
1247
1248
1249 private boolean matchesPattern( String path, List<String> patterns )
1250 {
1251 if ( CollectionUtils.isEmpty( patterns ) )
1252 {
1253 return false;
1254 }
1255
1256 if ( !path.startsWith( "/" ) )
1257 {
1258 path = "/" + path;
1259 }
1260
1261 for ( String pattern : patterns )
1262 {
1263 if ( !pattern.startsWith( "/" ) )
1264 {
1265 pattern = "/" + pattern;
1266 }
1267
1268 if ( SelectorUtils.matchPath( pattern, path, false ) )
1269 {
1270 return true;
1271 }
1272 }
1273
1274 return false;
1275 }
1276
1277
1278
1279
1280 @Override
1281 public List<ProxyConnector> getProxyConnectors( ManagedRepositoryContent repository )
1282 {
1283
1284 if ( !this.proxyConnectorMap.containsKey( repository.getId() ) )
1285 {
1286 return Collections.emptyList();
1287 }
1288 List<ProxyConnector> ret = new ArrayList<>( this.proxyConnectorMap.get( repository.getId() ) );
1289
1290 Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
1291 return ret;
1292
1293 }
1294
1295 @Override
1296 public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1297 {
1298 if ( ConfigurationNames.isNetworkProxy( propertyName )
1299 || ConfigurationNames.isManagedRepositories( propertyName )
1300 || ConfigurationNames.isRemoteRepositories( propertyName )
1301 || ConfigurationNames.isProxyConnector( propertyName ) )
1302 {
1303 initConnectorsAndNetworkProxies();
1304 }
1305 }
1306
1307 protected String addParameters( String path, RemoteRepository remoteRepository )
1308 {
1309 if ( remoteRepository.getExtraParameters().isEmpty() )
1310 {
1311 return path;
1312 }
1313
1314 boolean question = false;
1315
1316 StringBuilder res = new StringBuilder( path == null ? "" : path );
1317
1318 for ( Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
1319 {
1320 if ( !question )
1321 {
1322 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
1323 }
1324 }
1325
1326 return res.toString();
1327 }
1328
1329
1330 @Override
1331 public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1332 {
1333
1334 }
1335
1336 public ArchivaConfiguration getArchivaConfiguration()
1337 {
1338 return archivaConfiguration;
1339 }
1340
1341 public void setArchivaConfiguration( ArchivaConfiguration archivaConfiguration )
1342 {
1343 this.archivaConfiguration = archivaConfiguration;
1344 }
1345
1346 public RepositoryContentFactory getRepositoryFactory()
1347 {
1348 return repositoryFactory;
1349 }
1350
1351 public void setRepositoryFactory( RepositoryContentFactory repositoryFactory )
1352 {
1353 this.repositoryFactory = repositoryFactory;
1354 }
1355
1356 public MetadataTools getMetadataTools()
1357 {
1358 return metadataTools;
1359 }
1360
1361 public void setMetadataTools( MetadataTools metadataTools )
1362 {
1363 this.metadataTools = metadataTools;
1364 }
1365
1366 public UrlFailureCache getUrlFailureCache()
1367 {
1368 return urlFailureCache;
1369 }
1370
1371 public void setUrlFailureCache( UrlFailureCache urlFailureCache )
1372 {
1373 this.urlFailureCache = urlFailureCache;
1374 }
1375
1376 public WagonFactory getWagonFactory()
1377 {
1378 return wagonFactory;
1379 }
1380
1381 public void setWagonFactory( WagonFactory wagonFactory )
1382 {
1383 this.wagonFactory = wagonFactory;
1384 }
1385
1386 public Map<String, PreDownloadPolicy> getPreDownloadPolicies()
1387 {
1388 return preDownloadPolicies;
1389 }
1390
1391 public void setPreDownloadPolicies( Map<String, PreDownloadPolicy> preDownloadPolicies )
1392 {
1393 this.preDownloadPolicies = preDownloadPolicies;
1394 }
1395
1396 public Map<String, PostDownloadPolicy> getPostDownloadPolicies()
1397 {
1398 return postDownloadPolicies;
1399 }
1400
1401 public void setPostDownloadPolicies( Map<String, PostDownloadPolicy> postDownloadPolicies )
1402 {
1403 this.postDownloadPolicies = postDownloadPolicies;
1404 }
1405
1406 public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies()
1407 {
1408 return downloadErrorPolicies;
1409 }
1410
1411 public void setDownloadErrorPolicies( Map<String, DownloadErrorPolicy> downloadErrorPolicies )
1412 {
1413 this.downloadErrorPolicies = downloadErrorPolicies;
1414 }
1415 }