This project has retired. For details please refer to its
Attic page.
1 package org.apache.archiva.indexer.maven;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import org.apache.archiva.common.utils.PathUtil;
23 import org.apache.archiva.configuration.ArchivaConfiguration;
24 import org.apache.archiva.indexer.ArchivaIndexManager;
25 import org.apache.archiva.indexer.ArchivaIndexingContext;
26 import org.apache.archiva.indexer.IndexCreationFailedException;
27 import org.apache.archiva.indexer.IndexUpdateFailedException;
28 import org.apache.archiva.indexer.UnsupportedBaseContextException;
29 import org.apache.archiva.proxy.ProxyRegistry;
30 import org.apache.archiva.proxy.maven.WagonFactory;
31 import org.apache.archiva.proxy.maven.WagonFactoryException;
32 import org.apache.archiva.proxy.maven.WagonFactoryRequest;
33 import org.apache.archiva.proxy.model.NetworkProxy;
34 import org.apache.archiva.repository.EditableRepository;
35 import org.apache.archiva.repository.ManagedRepository;
36 import org.apache.archiva.repository.base.PasswordCredentials;
37 import org.apache.archiva.repository.RemoteRepository;
38 import org.apache.archiva.repository.Repository;
39 import org.apache.archiva.repository.RepositoryType;
40 import org.apache.archiva.repository.UnsupportedRepositoryTypeException;
41 import org.apache.archiva.repository.storage.FilesystemStorage;
42 import org.apache.archiva.repository.storage.RepositoryStorage;
43 import org.apache.archiva.repository.storage.StorageAsset;
44 import org.apache.archiva.repository.features.IndexCreationFeature;
45 import org.apache.archiva.repository.features.RemoteIndexFeature;
46 import org.apache.archiva.repository.storage.StorageUtil;
47 import org.apache.commons.lang3.StringUtils;
48 import org.apache.maven.index.ArtifactContext;
49 import org.apache.maven.index.ArtifactContextProducer;
50 import org.apache.maven.index.DefaultScannerListener;
51 import org.apache.maven.index.Indexer;
52 import org.apache.maven.index.IndexerEngine;
53 import org.apache.maven.index.Scanner;
54 import org.apache.maven.index.ScanningRequest;
55 import org.apache.maven.index.ScanningResult;
56 import org.apache.maven.index.context.ContextMemberProvider;
57 import org.apache.maven.index.context.IndexCreator;
58 import org.apache.maven.index.context.IndexingContext;
59 import org.apache.maven.index.context.StaticContextMemberProvider;
60 import org.apache.maven.index.packer.IndexPacker;
61 import org.apache.maven.index.packer.IndexPackingRequest;
62 import org.apache.maven.index.updater.IndexUpdateRequest;
63 import org.apache.maven.index.updater.IndexUpdater;
64 import org.apache.maven.index.updater.ResourceFetcher;
65 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
66 import org.apache.maven.wagon.ConnectionException;
67 import org.apache.maven.wagon.ResourceDoesNotExistException;
68 import org.apache.maven.wagon.StreamWagon;
69 import org.apache.maven.wagon.TransferFailedException;
70 import org.apache.maven.wagon.Wagon;
71 import org.apache.maven.wagon.authentication.AuthenticationException;
72 import org.apache.maven.wagon.authentication.AuthenticationInfo;
73 import org.apache.maven.wagon.authorization.AuthorizationException;
74 import org.apache.maven.wagon.events.TransferEvent;
75 import org.apache.maven.wagon.events.TransferListener;
76 import org.apache.maven.wagon.proxy.ProxyInfo;
77 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
78 import org.apache.maven.wagon.shared.http.HttpConfiguration;
79 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
80 import org.slf4j.Logger;
81 import org.slf4j.LoggerFactory;
82 import org.springframework.stereotype.Service;
83
84 import javax.inject.Inject;
85 import java.io.FileNotFoundException;
86 import java.io.IOException;
87 import java.io.InputStream;
88 import java.net.MalformedURLException;
89 import java.net.URI;
90 import java.nio.file.Files;
91 import java.nio.file.Path;
92 import java.nio.file.Paths;
93 import java.util.Collection;
94 import java.util.List;
95 import java.util.Map;
96 import java.util.Objects;
97 import java.util.concurrent.ConcurrentSkipListSet;
98 import java.util.stream.Collectors;
99
100
101
102
103
104
105
106
107 @Service( "archivaIndexManager#maven" )
108 public class MavenIndexManager implements ArchivaIndexManager {
109
110 private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class );
111
112 @Inject
113 private Indexer indexer;
114
115 @Inject
116 private IndexerEngine indexerEngine;
117
118 @Inject
119 private List<? extends IndexCreator> indexCreators;
120
121 @Inject
122 private IndexPacker indexPacker;
123
124 @Inject
125 private Scanner scanner;
126
127 @Inject
128 private ArchivaConfiguration archivaConfiguration;
129
130 @Inject
131 private WagonFactory wagonFactory;
132
133 @Inject
134 private IndexUpdater indexUpdater;
135
136 @Inject
137 private ArtifactContextProducer artifactContextProducer;
138
139 @Inject
140 private ProxyRegistry proxyRegistry;
141
142
143 private ConcurrentSkipListSet<StorageAsset> activeContexts = new ConcurrentSkipListSet<>( );
144
145 private static final int WAIT_TIME = 100;
146 private static final int MAX_WAIT = 10;
147
148
149 public static IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException
150 {
151 if (context!=null)
152 {
153 if ( !context.supports( IndexingContext.class ) )
154 {
155 log.error( "The provided archiva index context does not support the maven IndexingContext" );
156 throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
157 }
158 return context.getBaseContext( IndexingContext.class );
159 } else {
160 return null;
161 }
162 }
163
164 private StorageAsset getIndexPath( ArchivaIndexingContext ctx )
165 {
166 return ctx.getPath( );
167 }
168
169 @FunctionalInterface
170 interface IndexUpdateConsumer
171 {
172
173 void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
174 }
175
176
177
178
179
180 private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
181 {
182 if (context==null) {
183 throw new IndexUpdateFailedException( "Given context is null" );
184 }
185 IndexingContext indexingContext = null;
186 try
187 {
188 indexingContext = getMvnContext( context );
189 }
190 catch ( UnsupportedBaseContextException e )
191 {
192 throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
193 }
194 final StorageAsset ctxPath = getIndexPath( context );
195 int loop = MAX_WAIT;
196 boolean active = false;
197 while ( loop-- > 0 && !active )
198 {
199 active = activeContexts.add( ctxPath );
200 try
201 {
202 Thread.currentThread( ).sleep( WAIT_TIME );
203 }
204 catch ( InterruptedException e )
205 {
206
207 }
208 }
209 if ( active )
210 {
211 try
212 {
213 function.accept( indexingContext );
214 }
215 finally
216 {
217 activeContexts.remove( ctxPath );
218 }
219 }
220 else
221 {
222 throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
223 }
224 }
225
226 @Override
227 public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
228 {
229 executeUpdateFunction( context, indexingContext -> {
230 try
231 {
232 IndexPackingRequest request = new IndexPackingRequest( indexingContext,
233 indexingContext.acquireIndexSearcher( ).getIndexReader( ),
234 indexingContext.getIndexDirectoryFile( ) );
235 indexPacker.packIndex( request );
236 indexingContext.updateTimestamp( true );
237 }
238 catch ( IOException e )
239 {
240 log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
241 throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
242 }
243 }
244 );
245
246 }
247
248 @Override
249 public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
250 {
251 executeUpdateFunction( context, indexingContext -> {
252 DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
253 ScanningRequest request = new ScanningRequest( indexingContext, listener );
254 ScanningResult result = scanner.scan( request );
255 if ( result.hasExceptions( ) )
256 {
257 log.error( "Exceptions occured during index scan of " + context.getId( ) );
258 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
259 s -> log.error( "Message: " + s )
260 );
261 }
262
263 } );
264 }
265
266 @Override
267 public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
268 {
269 log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
270 URI remoteUpdateUri;
271 if ( !( context.getRepository( ) instanceof RemoteRepository ) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
272 {
273 throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
274 } else {
275 RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
276 remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
277 }
278 final RemoteRepositorypache/archiva/repository/RemoteRepository.html#RemoteRepository">RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
279
280 executeUpdateFunction( context,
281 indexingContext -> {
282 try
283 {
284
285 Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
286 Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
287 Files.createDirectories( indexCacheDirectory );
288 if ( Files.exists( tempIndexDirectory ) )
289 {
290 org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
291 }
292 Files.createDirectories( tempIndexDirectory );
293 tempIndexDirectory.toFile( ).deleteOnExit( );
294 String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
295
296 String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
297
298 NetworkProxy networkProxy = null;
299 if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
300 {
301 RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
302 if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
303 {
304 networkProxy = proxyRegistry.getNetworkProxy( rif.getProxyId( ) );
305 if ( networkProxy == null )
306 {
307 log.warn(
308 "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
309 rif.getProxyId( ) );
310 }
311 }
312
313 final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
314 new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
315 networkProxy )
316 );
317 int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
318 wagon.setReadTimeout( readTimeout );
319 wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
320
321 if ( wagon instanceof AbstractHttpClientWagon )
322 {
323 HttpConfiguration httpConfiguration = new HttpConfiguration( );
324 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
325 httpMethodConfiguration.setUsePreemptive( true );
326 httpMethodConfiguration.setReadTimeout( readTimeout );
327 httpConfiguration.setGet( httpMethodConfiguration );
328 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
329 }
330
331 wagon.addTransferListener( new DownloadListener( ) );
332 ProxyInfo proxyInfo = null;
333 if ( networkProxy != null )
334 {
335 proxyInfo = new ProxyInfo( );
336 proxyInfo.setType( networkProxy.getProtocol( ) );
337 proxyInfo.setHost( networkProxy.getHost( ) );
338 proxyInfo.setPort( networkProxy.getPort( ) );
339 proxyInfo.setUserName( networkProxy.getUsername( ) );
340 proxyInfo.setPassword( new String(networkProxy.getPassword( )) );
341 }
342 AuthenticationInfo authenticationInfo = null;
343 if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) )
344 {
345 PasswordCredentials../org/apache/archiva/repository/base/PasswordCredentials.html#PasswordCredentials">PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
346 authenticationInfo = new AuthenticationInfo( );
347 authenticationInfo.setUserName( creds.getUsername( ) );
348 authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
349 }
350 wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
351 proxyInfo );
352
353 Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
354 if ( !Files.exists( indexDirectory ) )
355 {
356 Files.createDirectories( indexDirectory );
357 }
358
359 ResourceFetcher resourceFetcher =
360 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
361 IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
362 request.setForceFullUpdate( fullUpdate );
363 request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
364
365 indexUpdater.fetchAndUpdateIndex( request );
366
367 indexingContext.updateTimestamp( true );
368 }
369
370 }
371 catch ( AuthenticationException e )
372 {
373 log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
374 throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
375 }
376 catch ( ConnectionException e )
377 {
378 log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
379 throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
380 }
381 catch ( MalformedURLException e )
382 {
383 log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
384 throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
385 }
386 catch ( IOException e )
387 {
388 log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
389 throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
390 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
391 }
392 catch ( WagonFactoryException e )
393 {
394 log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
395 throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
396 }
397 } );
398
399 }
400
401 @Override
402 public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
403 {
404 final StorageAsset ctxUri = context.getPath();
405 executeUpdateFunction(context, indexingContext -> {
406 Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
407 try {
408 indexer.addArtifactsToIndex(artifacts, indexingContext);
409 } catch (IOException e) {
410 log.error("IOException while adding artifact {}", e.getMessage(), e);
411 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
412 + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
413 }
414 });
415 }
416
417 @Override
418 public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
419 {
420 final StorageAsset ctxUri = context.getPath();
421 executeUpdateFunction(context, indexingContext -> {
422 Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
423 try {
424 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
425 } catch (IOException e) {
426 log.error("IOException while removing artifact {}", e.getMessage(), e);
427 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
428 + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
429 }
430 });
431
432 }
433
434 @Override
435 public boolean supportsRepository( RepositoryType type )
436 {
437 return type == RepositoryType.MAVEN;
438 }
439
440 @Override
441 public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
442 {
443 log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
444 if ( repository.getType( ) != RepositoryType.MAVEN )
445 {
446 throw new UnsupportedRepositoryTypeException( repository.getType( ) );
447 }
448 IndexingContext mvnCtx = null;
449 try
450 {
451 if ( repository instanceof RemoteRepository )
452 {
453 mvnCtx = createRemoteContext( (RemoteRepository) repository );
454 }
455 else if ( repository instanceof ManagedRepository )
456 {
457 mvnCtx = createManagedContext( (ManagedRepository) repository );
458 }
459 }
460 catch ( IOException e )
461 {
462 log.error( "IOException during context creation " + e.getMessage( ), e );
463 throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
464 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
465 }
466
467 return new MavenIndexContext( repository, mvnCtx );
468 }
469
470 @Override
471 public ArchivaIndexingContext../org/apache/archiva/indexer/ArchivaIndexingContext.html#ArchivaIndexingContext">ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
472 ArchivaIndexingContext ctx;
473 executeUpdateFunction(context, indexingContext -> {
474 try {
475 indexingContext.close(true);
476 } catch (IOException e) {
477 log.warn("Index close failed");
478 }
479 try {
480 StorageUtil.deleteRecursively(context.getPath());
481 } catch (IOException e) {
482 throw new IndexUpdateFailedException("Could not delete index files");
483 }
484 });
485 try {
486 Repository repo = context.getRepository();
487 ctx = createContext(context.getRepository());
488 if (repo instanceof EditableRepository) {
489 ((EditableRepository)repo).setIndexingContext(ctx);
490 }
491 } catch (IndexCreationFailedException e) {
492 throw new IndexUpdateFailedException("Could not create index");
493 }
494 return ctx;
495 }
496
497 @Override
498 public ArchivaIndexingContext/../org/apache/archiva/indexer/ArchivaIndexingContext.html#ArchivaIndexingContext">ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
499 if (context==null) {
500 return null;
501 }
502 if (context.supports(IndexingContext.class)) {
503 try {
504 StorageAsset newPath = getIndexPath(repo);
505 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
506 Path oldPath = ctx.getIndexDirectoryFile().toPath();
507 if (oldPath.equals(newPath)) {
508
509 return context;
510 }
511 if (!Files.exists(oldPath)) {
512 return createContext(repo);
513 } else if (context.isEmpty()) {
514 context.close();
515 return createContext(repo);
516 } else {
517 context.close(false);
518 Files.move(oldPath, newPath.getFilePath());
519 return createContext(repo);
520 }
521 } catch (IOException e) {
522 log.error("IOException while moving index directory {}", e.getMessage(), e);
523 throw new IndexCreationFailedException("Could not recreated the index.", e);
524 } catch (UnsupportedBaseContextException e) {
525 throw new IndexCreationFailedException("The given context, is not a maven context.");
526 }
527 } else {
528 throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
529 }
530 }
531
532 @Override
533 public void updateLocalIndexPath(Repository repo) {
534 if (repo.supportsFeature(IndexCreationFeature.class)) {
535 IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
536 try {
537 icf.setLocalIndexPath(getIndexPath(repo));
538 icf.setLocalPackedIndexPath(getPackedIndexPath(repo));
539 } catch (IOException e) {
540 log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
541 }
542 }
543 }
544
545 @Override
546 public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts,
547 boolean packIndex) throws UnsupportedOperationException,
548 IndexCreationFailedException, IllegalArgumentException {
549 if (!destinationRepo.supportsFeature(IndexCreationFeature.class)) {
550 throw new IllegalArgumentException("The given repository does not support the indexcreation feature");
551 }
552 Path mergedIndexDirectory = null;
553 try {
554 mergedIndexDirectory = Files.createTempDirectory("archivaMergedIndex");
555 } catch (IOException e) {
556 log.error("Could not create temporary directory for merged index: {}", e.getMessage(), e);
557 throw new IndexCreationFailedException("IO error while creating temporary directory for merged index: "+e.getMessage(), e);
558 }
559 IndexCreationFeature indexCreationFeature = destinationRepo.getFeature(IndexCreationFeature.class).get();
560 if (indexCreationFeature.getLocalIndexPath()== null) {
561 throw new IllegalArgumentException("The given repository does not have a local index path");
562 }
563 StorageAsset destinationPath = indexCreationFeature.getLocalIndexPath();
564
565 String tempRepoId = mergedIndexDirectory.getFileName().toString();
566
567 try
568 {
569 Path indexLocation = destinationPath.getFilePath();
570
571 List<IndexingContext> members = contexts.stream( ).filter(ctx -> ctx.supports(IndexingContext.class)).map( ctx ->
572 {
573 try {
574 return ctx.getBaseContext(IndexingContext.class);
575 } catch (UnsupportedBaseContextException e) {
576
577 return null;
578 }
579 }).filter( Objects::nonNull ).collect( Collectors.toList() );
580 ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
581 IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
582 indexLocation.toFile(), true, memberProvider);
583 mergedCtx.optimize();
584
585 if ( packIndex )
586 {
587 IndexPackingRequest request = new IndexPackingRequest( mergedCtx,
588 mergedCtx.acquireIndexSearcher().getIndexReader(),
589 indexLocation.toFile() );
590 indexPacker.packIndex( request );
591 }
592
593 return new MavenIndexContext(destinationRepo, mergedCtx);
594 }
595 catch ( IOException e)
596 {
597 throw new IndexCreationFailedException( "IO Error during index merge: "+ e.getMessage(), e );
598 }
599 }
600
601 private StorageAsset getIndexPath(URI indexDirUri, RepositoryStorage repoStorage, String defaultDir) throws IOException
602 {
603 StorageAsset rootAsset = repoStorage.getAsset("");
604 RepositoryStorage storage = rootAsset.getStorage();
605 Path indexDirectory;
606 Path repositoryPath = rootAsset.getFilePath().toAbsolutePath();
607 StorageAsset indexDir;
608 if ( ! StringUtils.isEmpty(indexDirUri.toString( ) ) )
609 {
610
611 indexDirectory = PathUtil.getPathFromUri( indexDirUri );
612
613 if ( indexDirectory.isAbsolute( ) && !indexDirectory.startsWith(repositoryPath))
614 {
615 if (storage instanceof FilesystemStorage) {
616 FilesystemStorage/org/apache/archiva/repository/storage/FilesystemStorage.html#FilesystemStorage">FilesystemStorage fsStorage = (FilesystemStorage) storage;
617 FilesystemStoragesystemStorage.html#FilesystemStorage">FilesystemStorage indexStorage = new FilesystemStorage(indexDirectory.getParent(), fsStorage.getFileLockManager());
618 indexDir = indexStorage.getAsset(indexDirectory.getFileName().toString());
619 } else {
620 throw new IOException("The given storage is not file based.");
621 }
622 } else if (indexDirectory.isAbsolute()) {
623 indexDir = storage.getAsset(repositoryPath.relativize(indexDirectory).toString());
624 }
625 else
626 {
627 indexDir = storage.getAsset(indexDirectory.toString());
628 }
629 }
630 else
631 {
632 indexDir = storage.getAsset( defaultDir );
633 }
634
635 if ( !indexDir.exists() )
636 {
637 indexDir = storage.addAsset(indexDir.getPath(), true);
638 }
639 return indexDir;
640 }
641
642 private StorageAsset getIndexPath( Repository repo) throws IOException {
643 IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
644 return getIndexPath( icf.getIndexPath(), repo, DEFAULT_INDEX_PATH);
645 }
646
647 private StorageAsset getPackedIndexPath(Repository repo) throws IOException {
648 IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
649 return getIndexPath(icf.getPackedIndexPath(), repo, DEFAULT_PACKED_INDEX_PATH);
650 }
651
652 private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
653 {
654 String contextKey = "remote-" + remoteRepository.getId( );
655
656
657
658 Path repoDir = remoteRepository.getAsset( "" ).getFilePath();
659 if ( !Files.exists( repoDir ) )
660 {
661 Files.createDirectories( repoDir );
662 }
663
664 StorageAsset indexDirectory;
665
666
667 if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
668 {
669 RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
670 indexDirectory = getIndexPath(remoteRepository);
671 String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
672 try
673 {
674
675 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
676 }
677 catch ( IndexFormatTooOldException e )
678 {
679
680
681 log.warn( "the index of repository {} is too old we have to delete and recreate it",
682 remoteRepository.getId( ) );
683 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
684 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
685
686 }
687 }
688 else
689 {
690 throw new IOException( "No remote index defined" );
691 }
692 }
693
694 private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, StorageAsset indexDirectory, String indexUrl ) throws IOException
695 {
696 try
697 {
698 if (!Files.exists(indexDirectory.getFilePath())) {
699 Files.createDirectories(indexDirectory.getFilePath());
700 }
701 return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.getFilePath( ).toFile( ),
702 repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
703 indexUrl,
704 true, false,
705 indexCreators );
706 } catch (Exception e) {
707 log.error("Could not create index for asset {}", indexDirectory);
708 throw new IOException(e);
709 }
710 }
711
712 private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
713 {
714
715 IndexingContext context;
716
717 Path repositoryDirectory = repository.getAsset( "" ).getFilePath();
718
719 if ( !Files.exists( repositoryDirectory ) )
720 {
721 try
722 {
723 Files.createDirectories( repositoryDirectory );
724 }
725 catch ( IOException e )
726 {
727 log.error( "Could not create directory {}", repositoryDirectory );
728 }
729 }
730
731 StorageAsset indexDirectory;
732
733 if ( repository.supportsFeature( IndexCreationFeature.class ) )
734 {
735 indexDirectory = getIndexPath(repository);
736 log.debug( "Preparing index at {}", indexDirectory );
737
738 String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
739 try
740 {
741 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
742 context.setSearchable( repository.isScanned( ) );
743 }
744 catch ( IndexFormatTooOldException e )
745 {
746
747
748 log.warn( "the index of repository {} is too old we have to delete and recreate it",
749 repository.getId( ) );
750 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
751 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
752 context.setSearchable( repository.isScanned( ) );
753 }
754 return context;
755 }
756 else
757 {
758 throw new IOException( "No repository index defined" );
759 }
760 }
761
762 private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
763 {
764 if ( rif.getIndexUri( ) == null )
765 {
766 return baseUri.resolve( DEFAULT_INDEX_PATH ).toString( );
767 }
768 else
769 {
770 return baseUri.resolve( rif.getIndexUri( ) ).toString( );
771 }
772 }
773
774 private static final class DownloadListener
775 implements TransferListener
776 {
777 private Logger log = LoggerFactory.getLogger( getClass( ) );
778
779 private String resourceName;
780
781 private long startTime;
782
783 private int totalLength = 0;
784
785 @Override
786 public void transferInitiated( TransferEvent transferEvent )
787 {
788 startTime = System.currentTimeMillis( );
789 resourceName = transferEvent.getResource( ).getName( );
790 log.debug( "initiate transfer of {}", resourceName );
791 }
792
793 @Override
794 public void transferStarted( TransferEvent transferEvent )
795 {
796 this.totalLength = 0;
797 resourceName = transferEvent.getResource( ).getName( );
798 log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
799 }
800
801 @Override
802 public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
803 {
804 log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
805 this.totalLength += length;
806 }
807
808 @Override
809 public void transferCompleted( TransferEvent transferEvent )
810 {
811 resourceName = transferEvent.getResource( ).getName( );
812 long endTime = System.currentTimeMillis( );
813 log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
814 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
815 }
816
817 @Override
818 public void transferError( TransferEvent transferEvent )
819 {
820 log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
821 transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
822 }
823
824 @Override
825 public void debug( String message )
826 {
827 log.debug( "transfer debug {}", message );
828 }
829 }
830
831 private static class WagonResourceFetcher
832 implements ResourceFetcher
833 {
834
835 Logger log;
836
837 Path tempIndexDirectory;
838
839 Wagon wagon;
840
841 RemoteRepository remoteRepository;
842
843 private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
844 RemoteRepository remoteRepository )
845 {
846 this.log = log;
847 this.tempIndexDirectory = tempIndexDirectory;
848 this.wagon = wagon;
849 this.remoteRepository = remoteRepository;
850 }
851
852 @Override
853 public void connect( String id, String url ) {
854
855 }
856
857 @Override
858 public void disconnect( ) {
859
860 }
861
862 @Override
863 public InputStream retrieve( String name )
864 throws IOException {
865 try
866 {
867 log.info( "index update retrieve file, name:{}", name );
868 Path file = tempIndexDirectory.resolve( name );
869 Files.deleteIfExists( file );
870 file.toFile( ).deleteOnExit( );
871 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
872 return Files.newInputStream( file );
873 }
874 catch ( AuthorizationException | TransferFailedException e )
875 {
876 throw new IOException( e.getMessage( ), e );
877 }
878 catch ( ResourceDoesNotExistException e )
879 {
880 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
881 fnfe.initCause( e );
882 throw fnfe;
883 }
884 }
885
886
887 protected String addParameters( String path, RemoteRepository remoteRepository )
888 {
889 if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
890 {
891 return path;
892 }
893
894 boolean question = false;
895
896 StringBuilder res = new StringBuilder( path == null ? "" : path );
897
898 for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
899 {
900 if ( !question )
901 {
902 res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
903 }
904 }
905
906 return res.toString( );
907 }
908
909 }
910 }