Source code

001package org.apache.archiva.indexer.maven;
002
003/*
004 * Licensed to the Apache Software Foundation (ASF) under one
005 * or more contributor license agreements.  See the NOTICE file
006 * distributed with this work for additional information
007 * regarding copyright ownership.  The ASF licenses this file
008 * to you under the Apache License, Version 2.0 (the
009 * "License"); you may not use this file except in compliance
010 * with the License.  You may obtain a copy of the License at
011 *
012 *  http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing,
015 * software distributed under the License is distributed on an
016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
017 * KIND, either express or implied.  See the License for the
018 * specific language governing permissions and limitations
019 * under the License.
020 */
021
022import org.apache.archiva.common.utils.PathUtil;
023import org.apache.archiva.configuration.ArchivaConfiguration;
024import org.apache.archiva.indexer.ArchivaIndexManager;
025import org.apache.archiva.indexer.ArchivaIndexingContext;
026import org.apache.archiva.indexer.IndexCreationFailedException;
027import org.apache.archiva.indexer.IndexUpdateFailedException;
028import org.apache.archiva.indexer.UnsupportedBaseContextException;
029import org.apache.archiva.proxy.ProxyRegistry;
030import org.apache.archiva.proxy.maven.WagonFactory;
031import org.apache.archiva.proxy.maven.WagonFactoryException;
032import org.apache.archiva.proxy.maven.WagonFactoryRequest;
033import org.apache.archiva.proxy.model.NetworkProxy;
034import org.apache.archiva.repository.EditableRepository;
035import org.apache.archiva.repository.ManagedRepository;
036import org.apache.archiva.repository.base.PasswordCredentials;
037import org.apache.archiva.repository.RemoteRepository;
038import org.apache.archiva.repository.Repository;
039import org.apache.archiva.repository.RepositoryType;
040import org.apache.archiva.repository.UnsupportedRepositoryTypeException;
041import org.apache.archiva.repository.storage.FilesystemStorage;
042import org.apache.archiva.repository.storage.RepositoryStorage;
043import org.apache.archiva.repository.storage.StorageAsset;
044import org.apache.archiva.repository.features.IndexCreationFeature;
045import org.apache.archiva.repository.features.RemoteIndexFeature;
046import org.apache.archiva.repository.storage.StorageUtil;
047import org.apache.commons.lang3.StringUtils;
048import org.apache.maven.index.ArtifactContext;
049import org.apache.maven.index.ArtifactContextProducer;
050import org.apache.maven.index.DefaultScannerListener;
051import org.apache.maven.index.Indexer;
052import org.apache.maven.index.IndexerEngine;
053import org.apache.maven.index.Scanner;
054import org.apache.maven.index.ScanningRequest;
055import org.apache.maven.index.ScanningResult;
056import org.apache.maven.index.context.ContextMemberProvider;
057import org.apache.maven.index.context.IndexCreator;
058import org.apache.maven.index.context.IndexingContext;
059import org.apache.maven.index.context.StaticContextMemberProvider;
060import org.apache.maven.index.packer.IndexPacker;
061import org.apache.maven.index.packer.IndexPackingRequest;
062import org.apache.maven.index.updater.IndexUpdateRequest;
063import org.apache.maven.index.updater.IndexUpdater;
064import org.apache.maven.index.updater.ResourceFetcher;
065import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
066import org.apache.maven.wagon.ConnectionException;
067import org.apache.maven.wagon.ResourceDoesNotExistException;
068import org.apache.maven.wagon.StreamWagon;
069import org.apache.maven.wagon.TransferFailedException;
070import org.apache.maven.wagon.Wagon;
071import org.apache.maven.wagon.authentication.AuthenticationException;
072import org.apache.maven.wagon.authentication.AuthenticationInfo;
073import org.apache.maven.wagon.authorization.AuthorizationException;
074import org.apache.maven.wagon.events.TransferEvent;
075import org.apache.maven.wagon.events.TransferListener;
076import org.apache.maven.wagon.proxy.ProxyInfo;
077import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
078import org.apache.maven.wagon.shared.http.HttpConfiguration;
079import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
080import org.slf4j.Logger;
081import org.slf4j.LoggerFactory;
082import org.springframework.stereotype.Service;
083
084import javax.inject.Inject;
085import java.io.FileNotFoundException;
086import java.io.IOException;
087import java.io.InputStream;
088import java.net.MalformedURLException;
089import java.net.URI;
090import java.nio.file.Files;
091import java.nio.file.Path;
092import java.nio.file.Paths;
093import java.util.Collection;
094import java.util.List;
095import java.util.Map;
096import java.util.Objects;
097import java.util.concurrent.ConcurrentSkipListSet;
098import java.util.stream.Collectors;
099
100/**
101 * Maven implementation of index manager.
102 * The index manager is a singleton, so we try to make sure, that index operations are not running
103 * parallel by synchronizing on the index path.
104 * A update operation waits for parallel running methods to finish before starting, but after a certain
105 * time of retries a IndexUpdateFailedException is thrown.
106 */
107@Service( "archivaIndexManager#maven" )
108public class MavenIndexManager implements ArchivaIndexManager {
109
110    private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class );
111
112    @Inject
113    private Indexer indexer;
114
115    @Inject
116    private IndexerEngine indexerEngine;
117
118    @Inject
119    private List<? extends IndexCreator> indexCreators;
120
121    @Inject
122    private IndexPacker indexPacker;
123
124    @Inject
125    private Scanner scanner;
126
127    @Inject
128    private ArchivaConfiguration archivaConfiguration;
129
130    @Inject
131    private WagonFactory wagonFactory;
132
133    @Inject
134    private IndexUpdater indexUpdater;
135
136    @Inject
137    private ArtifactContextProducer artifactContextProducer;
138
139    @Inject
140    private ProxyRegistry proxyRegistry;
141
142
143    private ConcurrentSkipListSet<StorageAsset> activeContexts = new ConcurrentSkipListSet<>( );
144
145    private static final int WAIT_TIME = 100;
146    private static final int MAX_WAIT = 10;
147
148
149    public static IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException
150    {
151        if (context!=null)
152        {
153            if ( !context.supports( IndexingContext.class ) )
154            {
155                log.error( "The provided archiva index context does not support the maven IndexingContext" );
156                throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
157            }
158            return context.getBaseContext( IndexingContext.class );
159        } else {
160            return null;
161        }
162    }
163
164    private StorageAsset getIndexPath( ArchivaIndexingContext ctx )
165    {
166        return ctx.getPath( );
167    }
168
169    @FunctionalInterface
170    interface IndexUpdateConsumer
171    {
172
173        void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
174    }
175
176    /*
177     * This method is used to do some actions around the update execution code. And to make sure, that no other
178     * method is running on the same index.
179     */
180    private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
181    {
182        if (context==null) {
183            throw new IndexUpdateFailedException( "Given context is null" );
184        }
185        IndexingContext indexingContext = null;
186        try
187        {
188            indexingContext = getMvnContext( context );
189        }
190        catch ( UnsupportedBaseContextException e )
191        {
192            throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
193        }
194        final StorageAsset ctxPath = getIndexPath( context );
195        int loop = MAX_WAIT;
196        boolean active = false;
197        while ( loop-- > 0 && !active )
198        {
199            active = activeContexts.add( ctxPath );
200            try
201            {
202                Thread.currentThread( ).sleep( WAIT_TIME );
203            }
204            catch ( InterruptedException e )
205            {
206                // Ignore this
207            }
208        }
209        if ( active )
210        {
211            try
212            {
213                function.accept( indexingContext );
214            }
215            finally
216            {
217                activeContexts.remove( ctxPath );
218            }
219        }
220        else
221        {
222            throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
223        }
224    }
225
226    @Override
227    public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
228    {
229        executeUpdateFunction( context, indexingContext -> {
230                try
231                {
232                    IndexPackingRequest request = new IndexPackingRequest( indexingContext,
233                        indexingContext.acquireIndexSearcher( ).getIndexReader( ),
234                        indexingContext.getIndexDirectoryFile( ) );
235                    indexPacker.packIndex( request );
236                    indexingContext.updateTimestamp( true );
237                }
238                catch ( IOException e )
239                {
240                    log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
241                    throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
242                }
243            }
244        );
245
246    }
247
248    @Override
249    public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
250    {
251        executeUpdateFunction( context, indexingContext -> {
252            DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
253            ScanningRequest request = new ScanningRequest( indexingContext, listener );
254            ScanningResult result = scanner.scan( request );
255            if ( result.hasExceptions( ) )
256            {
257                log.error( "Exceptions occured during index scan of " + context.getId( ) );
258                result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
259                    s -> log.error( "Message: " + s )
260                );
261            }
262
263        } );
264    }
265
266    @Override
267    public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
268    {
269        log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
270        URI remoteUpdateUri;
271        if ( !( context.getRepository( ) instanceof RemoteRepository ) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
272        {
273            throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
274        } else {
275            RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
276            remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
277        }
278        final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
279
280        executeUpdateFunction( context,
281            indexingContext -> {
282                try
283                {
284                    // create a temp directory to download files
285                    Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
286                    Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
287                    Files.createDirectories( indexCacheDirectory );
288                    if ( Files.exists( tempIndexDirectory ) )
289                    {
290                        org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
291                    }
292                    Files.createDirectories( tempIndexDirectory );
293                    tempIndexDirectory.toFile( ).deleteOnExit( );
294                    String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
295
296                    String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
297
298                    NetworkProxy networkProxy = null;
299                    if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
300                    {
301                        RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
302                        if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
303                        {
304                            networkProxy = proxyRegistry.getNetworkProxy( rif.getProxyId( ) );
305                            if ( networkProxy == null )
306                            {
307                                log.warn(
308                                    "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
309                                    rif.getProxyId( ) );
310                            }
311                        }
312
313                        final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
314                            new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
315                                networkProxy )
316                        );
317                        int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
318                        wagon.setReadTimeout( readTimeout );
319                        wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
320
321                        if ( wagon instanceof AbstractHttpClientWagon )
322                        {
323                            HttpConfiguration httpConfiguration = new HttpConfiguration( );
324                            HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
325                            httpMethodConfiguration.setUsePreemptive( true );
326                            httpMethodConfiguration.setReadTimeout( readTimeout );
327                            httpConfiguration.setGet( httpMethodConfiguration );
328                            AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
329                        }
330
331                        wagon.addTransferListener( new DownloadListener( ) );
332                        ProxyInfo proxyInfo = null;
333                        if ( networkProxy != null )
334                        {
335                            proxyInfo = new ProxyInfo( );
336                            proxyInfo.setType( networkProxy.getProtocol( ) );
337                            proxyInfo.setHost( networkProxy.getHost( ) );
338                            proxyInfo.setPort( networkProxy.getPort( ) );
339                            proxyInfo.setUserName( networkProxy.getUsername( ) );
340                            proxyInfo.setPassword( new String(networkProxy.getPassword( )) );
341                        }
342                        AuthenticationInfo authenticationInfo = null;
343                        if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) )
344                        {
345                            PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
346                            authenticationInfo = new AuthenticationInfo( );
347                            authenticationInfo.setUserName( creds.getUsername( ) );
348                            authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
349                        }
350                        wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
351                            proxyInfo );
352
353                        Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
354                        if ( !Files.exists( indexDirectory ) )
355                        {
356                            Files.createDirectories( indexDirectory );
357                        }
358
359                        ResourceFetcher resourceFetcher =
360                            new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
361                        IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
362                        request.setForceFullUpdate( fullUpdate );
363                        request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
364
365                        indexUpdater.fetchAndUpdateIndex( request );
366
367                        indexingContext.updateTimestamp( true );
368                    }
369
370                }
371                catch ( AuthenticationException e )
372                {
373                    log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
374                    throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
375                }
376                catch ( ConnectionException e )
377                {
378                    log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
379                    throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
380                }
381                catch ( MalformedURLException e )
382                {
383                    log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
384                    throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
385                }
386                catch ( IOException e )
387                {
388                    log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
389                    throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
390                        + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
391                }
392                catch ( WagonFactoryException e )
393                {
394                    log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
395                    throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
396                }
397            } );
398
399    }
400
401    @Override
402    public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
403    {
404        final StorageAsset ctxUri = context.getPath();
405        executeUpdateFunction(context, indexingContext -> {
406            Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
407            try {
408                indexer.addArtifactsToIndex(artifacts, indexingContext);
409            } catch (IOException e) {
410                log.error("IOException while adding artifact {}", e.getMessage(), e);
411                throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
412                + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
413            }
414        });
415    }
416
417    @Override
418    public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
419    {
420        final StorageAsset ctxUri = context.getPath();
421        executeUpdateFunction(context, indexingContext -> {
422            Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
423            try {
424                indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
425            } catch (IOException e) {
426                log.error("IOException while removing artifact {}", e.getMessage(), e);
427                throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
428                        + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
429            }
430        });
431
432    }
433
434    @Override
435    public boolean supportsRepository( RepositoryType type )
436    {
437        return type == RepositoryType.MAVEN;
438    }
439
440    @Override
441    public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
442    {
443        log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
444        if ( repository.getType( ) != RepositoryType.MAVEN )
445        {
446            throw new UnsupportedRepositoryTypeException( repository.getType( ) );
447        }
448        IndexingContext mvnCtx = null;
449        try
450        {
451            if ( repository instanceof RemoteRepository )
452            {
453                mvnCtx = createRemoteContext( (RemoteRepository) repository );
454            }
455            else if ( repository instanceof ManagedRepository )
456            {
457                mvnCtx = createManagedContext( (ManagedRepository) repository );
458            }
459        }
460        catch ( IOException e )
461        {
462            log.error( "IOException during context creation " + e.getMessage( ), e );
463            throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
464                + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
465        }
466
467        return new MavenIndexContext( repository, mvnCtx );
468    }
469
470    @Override
471    public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
472        ArchivaIndexingContext ctx;
473        executeUpdateFunction(context, indexingContext -> {
474            try {
475                indexingContext.close(true);
476            } catch (IOException e) {
477                log.warn("Index close failed");
478            }
479            try {
480                StorageUtil.deleteRecursively(context.getPath());
481            } catch (IOException e) {
482                throw new IndexUpdateFailedException("Could not delete index files");
483            }
484        });
485        try {
486            Repository repo = context.getRepository();
487            ctx = createContext(context.getRepository());
488            if (repo instanceof EditableRepository) {
489                ((EditableRepository)repo).setIndexingContext(ctx);
490            }
491        } catch (IndexCreationFailedException e) {
492            throw new IndexUpdateFailedException("Could not create index");
493        }
494        return ctx;
495    }
496
497    @Override
498    public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
499        if (context==null) {
500            return null;
501        }
502        if (context.supports(IndexingContext.class)) {
503            try {
504                StorageAsset newPath = getIndexPath(repo);
505                IndexingContext ctx = context.getBaseContext(IndexingContext.class);
506                Path oldPath = ctx.getIndexDirectoryFile().toPath();
507                if (oldPath.equals(newPath)) {
508                    // Nothing to do, if path does not change
509                    return context;
510                }
511                if (!Files.exists(oldPath)) {
512                    return createContext(repo);
513                } else if (context.isEmpty()) {
514                    context.close();
515                    return createContext(repo);
516                } else {
517                    context.close(false);
518                    Files.move(oldPath, newPath.getFilePath());
519                    return createContext(repo);
520                }
521            } catch (IOException e) {
522                log.error("IOException while moving index directory {}", e.getMessage(), e);
523                throw new IndexCreationFailedException("Could not recreated the index.", e);
524            } catch (UnsupportedBaseContextException e) {
525                throw new IndexCreationFailedException("The given context, is not a maven context.");
526            }
527        } else {
528            throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
529        }
530    }
531
532    @Override
533    public void updateLocalIndexPath(Repository repo) {
534        if (repo.supportsFeature(IndexCreationFeature.class)) {
535            IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
536            try {
537                icf.setLocalIndexPath(getIndexPath(repo));
538                icf.setLocalPackedIndexPath(getPackedIndexPath(repo));
539            } catch (IOException e) {
540                log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
541            }
542        }
543    }
544
545    @Override
546    public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts,
547                                                boolean packIndex) throws UnsupportedOperationException,
548            IndexCreationFailedException, IllegalArgumentException {
549        if (!destinationRepo.supportsFeature(IndexCreationFeature.class)) {
550            throw new IllegalArgumentException("The given repository does not support the indexcreation feature");
551        }
552        Path mergedIndexDirectory = null;
553        try {
554            mergedIndexDirectory = Files.createTempDirectory("archivaMergedIndex");
555        } catch (IOException e) {
556            log.error("Could not create temporary directory for merged index: {}", e.getMessage(), e);
557            throw new IndexCreationFailedException("IO error while creating temporary directory for merged index: "+e.getMessage(), e);
558        }
559        IndexCreationFeature indexCreationFeature = destinationRepo.getFeature(IndexCreationFeature.class).get();
560        if (indexCreationFeature.getLocalIndexPath()== null) {
561            throw new IllegalArgumentException("The given repository does not have a local index path");
562        }
563        StorageAsset destinationPath = indexCreationFeature.getLocalIndexPath();
564
565        String tempRepoId = mergedIndexDirectory.getFileName().toString();
566
567        try
568        {
569            Path indexLocation = destinationPath.getFilePath();
570
571            List<IndexingContext> members = contexts.stream( ).filter(ctx -> ctx.supports(IndexingContext.class)).map( ctx ->
572            {
573                try {
574                    return ctx.getBaseContext(IndexingContext.class);
575                } catch (UnsupportedBaseContextException e) {
576                    // does not happen here
577                    return null;
578                }
579            }).filter( Objects::nonNull ).collect( Collectors.toList() );
580            ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
581            IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
582                    indexLocation.toFile(), true, memberProvider);
583            mergedCtx.optimize();
584
585            if ( packIndex )
586            {
587                IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
588                        mergedCtx.acquireIndexSearcher().getIndexReader(), //
589                        indexLocation.toFile() );
590                indexPacker.packIndex( request );
591            }
592
593            return new MavenIndexContext(destinationRepo, mergedCtx);
594        }
595        catch ( IOException e)
596        {
597            throw new IndexCreationFailedException( "IO Error during index merge: "+ e.getMessage(), e );
598        }
599    }
600
601    private StorageAsset getIndexPath(URI indexDirUri, RepositoryStorage repoStorage, String defaultDir) throws IOException
602    {
603        StorageAsset rootAsset = repoStorage.getAsset("");
604        RepositoryStorage storage = rootAsset.getStorage();
605        Path indexDirectory;
606        Path repositoryPath = rootAsset.getFilePath().toAbsolutePath();
607        StorageAsset indexDir;
608        if ( ! StringUtils.isEmpty(indexDirUri.toString( ) ) )
609        {
610
611            indexDirectory = PathUtil.getPathFromUri( indexDirUri );
612            // not absolute so create it in repository directory
613            if ( indexDirectory.isAbsolute( ) && !indexDirectory.startsWith(repositoryPath))
614            {
615                if (storage instanceof FilesystemStorage) {
616                    FilesystemStorage fsStorage = (FilesystemStorage) storage;
617                    FilesystemStorage indexStorage = new FilesystemStorage(indexDirectory.getParent(), fsStorage.getFileLockManager());
618                    indexDir = indexStorage.getAsset(indexDirectory.getFileName().toString());
619                } else {
620                    throw new IOException("The given storage is not file based.");
621                }
622            } else if (indexDirectory.isAbsolute()) {
623                indexDir = storage.getAsset(repositoryPath.relativize(indexDirectory).toString());
624            }
625            else
626            {
627                indexDir = storage.getAsset(indexDirectory.toString());
628            }
629        }
630        else
631        {
632            indexDir = storage.getAsset( defaultDir );
633        }
634
635        if ( !indexDir.exists() )
636        {
637            indexDir = storage.addAsset(indexDir.getPath(), true);
638        }
639        return indexDir;
640    }
641
642    private StorageAsset getIndexPath( Repository repo) throws IOException {
643        IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
644        return getIndexPath( icf.getIndexPath(), repo, DEFAULT_INDEX_PATH);
645    }
646
647    private StorageAsset getPackedIndexPath(Repository repo) throws IOException {
648        IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
649        return getIndexPath(icf.getPackedIndexPath(), repo, DEFAULT_PACKED_INDEX_PATH);
650    }
651
652    private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
653    {
654        String contextKey = "remote-" + remoteRepository.getId( );
655
656
657        // create remote repository path
658        Path repoDir = remoteRepository.getAsset( "" ).getFilePath();
659        if ( !Files.exists( repoDir ) )
660        {
661            Files.createDirectories( repoDir );
662        }
663
664        StorageAsset indexDirectory;
665
666        // is there configured indexDirectory ?
667        if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
668        {
669            RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
670            indexDirectory = getIndexPath(remoteRepository);
671            String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
672            try
673            {
674
675                return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
676            }
677            catch ( IndexFormatTooOldException e )
678            {
679                // existing index with an old lucene format so we need to delete it!!!
680                // delete it first then recreate it.
681                log.warn( "the index of repository {} is too old we have to delete and recreate it", //
682                    remoteRepository.getId( ) );
683                org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
684                return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
685
686            }
687        }
688        else
689        {
690            throw new IOException( "No remote index defined" );
691        }
692    }
693
694    private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, StorageAsset indexDirectory, String indexUrl ) throws IOException
695    {
696        try
697        {
698            if (!Files.exists(indexDirectory.getFilePath())) {
699                Files.createDirectories(indexDirectory.getFilePath());
700            }
701            return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.getFilePath( ).toFile( ),
702                repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
703                indexUrl,
704                true, false,
705                indexCreators );
706        } catch (Exception e) {
707            log.error("Could not create index for asset {}", indexDirectory);
708            throw new IOException(e);
709        }
710    }
711
712    private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
713    {
714
715        IndexingContext context;
716        // take care first about repository location as can be relative
717        Path repositoryDirectory = repository.getAsset( "" ).getFilePath();
718
719        if ( !Files.exists( repositoryDirectory ) )
720        {
721            try
722            {
723                Files.createDirectories( repositoryDirectory );
724            }
725            catch ( IOException e )
726            {
727                log.error( "Could not create directory {}", repositoryDirectory );
728            }
729        }
730
731        StorageAsset indexDirectory;
732
733        if ( repository.supportsFeature( IndexCreationFeature.class ) )
734        {
735            indexDirectory = getIndexPath(repository);
736            log.debug( "Preparing index at {}", indexDirectory );
737
738            String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
739            try
740            {
741                context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
742                context.setSearchable( repository.isScanned( ) );
743            }
744            catch ( IndexFormatTooOldException e )
745            {
746                // existing index with an old lucene format so we need to delete it!!!
747                // delete it first then recreate it.
748                log.warn( "the index of repository {} is too old we have to delete and recreate it", //
749                    repository.getId( ) );
750                org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
751                context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
752                context.setSearchable( repository.isScanned( ) );
753            }
754            return context;
755        }
756        else
757        {
758            throw new IOException( "No repository index defined" );
759        }
760    }
761
762    private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
763    {
764        if ( rif.getIndexUri( ) == null )
765        {
766            return baseUri.resolve( DEFAULT_INDEX_PATH ).toString( );
767        }
768        else
769        {
770            return baseUri.resolve( rif.getIndexUri( ) ).toString( );
771        }
772    }
773
774    private static final class DownloadListener
775        implements TransferListener
776    {
777        private Logger log = LoggerFactory.getLogger( getClass( ) );
778
779        private String resourceName;
780
781        private long startTime;
782
783        private int totalLength = 0;
784
785        @Override
786        public void transferInitiated( TransferEvent transferEvent )
787        {
788            startTime = System.currentTimeMillis( );
789            resourceName = transferEvent.getResource( ).getName( );
790            log.debug( "initiate transfer of {}", resourceName );
791        }
792
793        @Override
794        public void transferStarted( TransferEvent transferEvent )
795        {
796            this.totalLength = 0;
797            resourceName = transferEvent.getResource( ).getName( );
798            log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
799        }
800
801        @Override
802        public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
803        {
804            log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
805            this.totalLength += length;
806        }
807
808        @Override
809        public void transferCompleted( TransferEvent transferEvent )
810        {
811            resourceName = transferEvent.getResource( ).getName( );
812            long endTime = System.currentTimeMillis( );
813            log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
814                this.totalLength / 1024, ( endTime - startTime ) / 1000 );
815        }
816
817        @Override
818        public void transferError( TransferEvent transferEvent )
819        {
820            log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
821                transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
822        }
823
824        @Override
825        public void debug( String message )
826        {
827            log.debug( "transfer debug {}", message );
828        }
829    }
830
831    private static class WagonResourceFetcher
832        implements ResourceFetcher
833    {
834
835        Logger log;
836
837        Path tempIndexDirectory;
838
839        Wagon wagon;
840
841        RemoteRepository remoteRepository;
842
843        private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
844                                      RemoteRepository remoteRepository )
845        {
846            this.log = log;
847            this.tempIndexDirectory = tempIndexDirectory;
848            this.wagon = wagon;
849            this.remoteRepository = remoteRepository;
850        }
851
852        @Override
853        public void connect( String id, String url ) {
854            //no op
855        }
856
857        @Override
858        public void disconnect( ) {
859            // no op
860        }
861
862        @Override
863        public InputStream retrieve( String name )
864            throws IOException {
865            try
866            {
867                log.info( "index update retrieve file, name:{}", name );
868                Path file = tempIndexDirectory.resolve( name );
869                Files.deleteIfExists( file );
870                file.toFile( ).deleteOnExit( );
871                wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
872                return Files.newInputStream( file );
873            }
874            catch ( AuthorizationException | TransferFailedException e )
875            {
876                throw new IOException( e.getMessage( ), e );
877            }
878            catch ( ResourceDoesNotExistException e )
879            {
880                FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
881                fnfe.initCause( e );
882                throw fnfe;
883            }
884        }
885
886        // FIXME remove crappy copy/paste
887        protected String addParameters( String path, RemoteRepository remoteRepository )
888        {
889            if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
890            {
891                return path;
892            }
893
894            boolean question = false;
895
896            StringBuilder res = new StringBuilder( path == null ? "" : path );
897
898            for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
899            {
900                if ( !question )
901                {
902                    res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
903                }
904            }
905
906            return res.toString( );
907        }
908
909    }
910}