Source code

001package org.apache.archiva.scheduler.indexing.maven;
002/*
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *   http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing,
014 * software distributed under the License is distributed on an
015 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
016 * KIND, either express or implied.  See the License for the
017 * specific language governing permissions and limitations
018 * under the License.
019 */
020
021import org.apache.archiva.proxy.maven.WagonFactory;
022import org.apache.archiva.proxy.maven.WagonFactoryRequest;
023import org.apache.archiva.proxy.model.NetworkProxy;
024import org.apache.archiva.repository.base.PasswordCredentials;
025import org.apache.archiva.repository.RemoteRepository;
026import org.apache.archiva.repository.RepositoryException;
027import org.apache.archiva.repository.RepositoryType;
028import org.apache.archiva.repository.features.RemoteIndexFeature;
029import org.apache.commons.lang3.time.StopWatch;
030import org.apache.maven.index.context.IndexingContext;
031import org.apache.maven.index.updater.IndexUpdateRequest;
032import org.apache.maven.index.updater.IndexUpdateResult;
033import org.apache.maven.index.updater.IndexUpdater;
034import org.apache.maven.index.updater.ResourceFetcher;
035import org.apache.maven.index_shaded.lucene.index.IndexNotFoundException;
036import org.apache.maven.wagon.ResourceDoesNotExistException;
037import org.apache.maven.wagon.StreamWagon;
038import org.apache.maven.wagon.TransferFailedException;
039import org.apache.maven.wagon.Wagon;
040import org.apache.maven.wagon.authentication.AuthenticationInfo;
041import org.apache.maven.wagon.authorization.AuthorizationException;
042import org.apache.maven.wagon.events.TransferEvent;
043import org.apache.maven.wagon.events.TransferListener;
044import org.apache.maven.wagon.proxy.ProxyInfo;
045import org.apache.maven.wagon.repository.Repository;
046import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
047import org.apache.maven.wagon.shared.http.HttpConfiguration;
048import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052import java.io.FileNotFoundException;
053import java.io.IOException;
054import java.io.InputStream;
055import java.nio.file.Files;
056import java.nio.file.Path;
057import java.nio.file.Paths;
058import java.util.List;
059import java.util.Map;
060
061/**
062 * @author Olivier Lamy
063 * @since 1.4-M1
064 */
065public class DownloadRemoteIndexTask
066    implements Runnable
067{
068    private Logger log = LoggerFactory.getLogger( getClass() );
069
070    private RemoteRepository remoteRepository;
071
072    private WagonFactory wagonFactory;
073
074    private NetworkProxy networkProxy;
075
076    private boolean fullDownload;
077
078    private List<String> runningRemoteDownloadIds;
079
080    private IndexUpdater indexUpdater;
081
082
083    public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
084                                    List<String> runningRemoteDownloadIds )
085    {
086        this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
087        this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
088        this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
089        this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
090        this.runningRemoteDownloadIds = runningRemoteDownloadIds;
091        this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
092    }
093
094    @Override
095    public void run()
096    {
097
098        // so short lock : not sure we need it
099        synchronized ( this.runningRemoteDownloadIds )
100        {
101            if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
102            {
103                // skip it as it's running
104                log.info( "skip download index remote for repo {} it's already running",
105                          this.remoteRepository.getId() );
106                return;
107            }
108            this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
109        }
110        Path tempIndexDirectory = null;
111        StopWatch stopWatch = new StopWatch();
112        stopWatch.start();
113        try
114        {
115            log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
116            if (this.remoteRepository.getIndexingContext()==null) {
117                throw new IndexNotFoundException("No index context set for repository "+remoteRepository.getId());
118            }
119            if (this.remoteRepository.getType()!= RepositoryType.MAVEN) {
120                throw new RepositoryException("Bad repository type");
121            }
122            if (!this.remoteRepository.supportsFeature(RemoteIndexFeature.class)) {
123                throw new RepositoryException("Repository does not support RemotIndexFeature "+remoteRepository.getId());
124            }
125            RemoteIndexFeature rif = this.remoteRepository.getFeature(RemoteIndexFeature.class).get();
126            IndexingContext indexingContext = this.remoteRepository.getIndexingContext().getBaseContext(IndexingContext.class);
127            // create a temp directory to download files
128            tempIndexDirectory = Paths.get(indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
129            Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
130            Files.createDirectories( indexCacheDirectory );
131            if ( Files.exists(tempIndexDirectory) )
132            {
133                org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
134            }
135            Files.createDirectories( tempIndexDirectory );
136            tempIndexDirectory.toFile().deleteOnExit();
137            String baseIndexUrl = indexingContext.getIndexUpdateUrl();
138
139            String wagonProtocol = this.remoteRepository.getLocation().getScheme();
140
141            final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
142                new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
143                    this.networkProxy )
144            );
145            // FIXME olamy having 2 config values
146            wagon.setReadTimeout( (int)rif.getDownloadTimeout().toMillis());
147            wagon.setTimeout( (int)remoteRepository.getTimeout().toMillis());
148
149            if ( wagon instanceof AbstractHttpClientWagon )
150            {
151                HttpConfiguration httpConfiguration = new HttpConfiguration();
152                HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
153                httpMethodConfiguration.setUsePreemptive( true );
154                httpMethodConfiguration.setReadTimeout( (int)rif.getDownloadTimeout().toMillis() );
155                httpConfiguration.setGet( httpMethodConfiguration );
156                AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
157            }
158
159            wagon.addTransferListener( new DownloadListener() );
160            ProxyInfo proxyInfo = null;
161            if ( this.networkProxy != null )
162            {
163                proxyInfo = new ProxyInfo();
164                proxyInfo.setType( this.networkProxy.getProtocol() );
165                proxyInfo.setHost( this.networkProxy.getHost() );
166                proxyInfo.setPort( this.networkProxy.getPort() );
167                proxyInfo.setUserName( this.networkProxy.getUsername() );
168                proxyInfo.setPassword( new String(this.networkProxy.getPassword()) );
169            }
170            AuthenticationInfo authenticationInfo = null;
171            if ( this.remoteRepository.getLoginCredentials()!=null && this.remoteRepository.getLoginCredentials() instanceof PasswordCredentials )
172            {
173                PasswordCredentials creds = (PasswordCredentials) this.remoteRepository.getLoginCredentials();
174                authenticationInfo = new AuthenticationInfo();
175                authenticationInfo.setUserName( creds.getUsername());
176                authenticationInfo.setPassword( new String(creds.getPassword()) );
177            }
178            log.debug("Connection to {}, authInfo={}", this.remoteRepository.getId(), authenticationInfo);
179            wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
180                           proxyInfo );
181
182            Path indexDirectory = indexingContext.getIndexDirectoryFile().toPath();
183            if ( !Files.exists(indexDirectory) )
184            {
185                Files.createDirectories( indexDirectory );
186            }
187            log.debug("Downloading index file to {}", indexDirectory);
188            log.debug("Index cache dir {}", indexCacheDirectory);
189
190            ResourceFetcher resourceFetcher =
191                new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
192            IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
193            request.setForceFullUpdate( this.fullDownload );
194            request.setLocalIndexCacheDir( indexCacheDirectory.toFile() );
195
196            IndexUpdateResult result = this.indexUpdater.fetchAndUpdateIndex(request);
197            log.debug("Update result success: {}", result.isSuccessful());
198            stopWatch.stop();
199            log.info( "time update index from remote for repository {}: {}ms", this.remoteRepository.getId(),
200                      ( stopWatch.getTime() ) );
201
202            // index packing optionnal ??
203            //IndexPackingRequest indexPackingRequest =
204            //    new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
205            //indexPacker.packIndex( indexPackingRequest );
206            indexingContext.updateTimestamp( true );
207
208        }
209        catch ( Exception e )
210        {
211            log.error( e.getMessage(), e );
212            throw new RuntimeException( e.getMessage(), e );
213        }
214        finally
215        {
216            deleteDirectoryQuiet( tempIndexDirectory );
217            this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
218        }
219        log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
220    }
221
222    private void deleteDirectoryQuiet( Path f )
223    {
224        try
225        {
226            org.apache.archiva.common.utils.FileUtils.deleteDirectory( f );
227        }
228        catch ( IOException e )
229        {
230            log.warn( "skip error delete {} : {}", f, e.getMessage() );
231        }
232    }
233
234
235    private static final class DownloadListener
236        implements TransferListener
237    {
238        private Logger log = LoggerFactory.getLogger( getClass() );
239
240        private String resourceName;
241
242        private long startTime;
243
244        private int totalLength = 0;
245
246        @Override
247        public void transferInitiated( TransferEvent transferEvent )
248        {
249            startTime = System.currentTimeMillis();
250            resourceName = transferEvent.getResource().getName();
251            log.debug( "initiate transfer of {}", resourceName );
252        }
253
254        @Override
255        public void transferStarted( TransferEvent transferEvent )
256        {
257            this.totalLength = 0;
258            resourceName = transferEvent.getResource().getName();
259            log.info("Transferring: {}, {}",  transferEvent.getResource().getContentLength(), transferEvent.getLocalFile().toString());
260            log.info( "start transfer of {}", transferEvent.getResource().getName() );
261        }
262
263        @Override
264        public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
265        {
266            log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
267            this.totalLength += length;
268        }
269
270        @Override
271        public void transferCompleted( TransferEvent transferEvent )
272        {
273            resourceName = transferEvent.getResource().getName();
274            long endTime = System.currentTimeMillis();
275            log.info( "end of transfer file {}: {}b, {}ms", transferEvent.getResource().getName(),
276                      this.totalLength, ( endTime - startTime ) );
277        }
278
279        @Override
280        public void transferError( TransferEvent transferEvent )
281        {
282            log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
283                      transferEvent.getException().getMessage(), transferEvent.getException() );
284        }
285
286        @Override
287        public void debug( String message )
288        {
289            log.debug( "transfer debug {}", message );
290        }
291    }
292
293    private static class WagonResourceFetcher
294        implements ResourceFetcher
295    {
296
297        Logger log;
298
299        Path tempIndexDirectory;
300
301        Wagon wagon;
302
303        RemoteRepository remoteRepository;
304
305        private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
306                                      RemoteRepository remoteRepository )
307        {
308            this.log = log;
309            this.tempIndexDirectory = tempIndexDirectory;
310            this.wagon = wagon;
311            this.remoteRepository = remoteRepository;
312        }
313
314        @Override
315        public void connect( String id, String url )
316            throws IOException
317        {
318            //no op  
319        }
320
321        @Override
322        public void disconnect()
323            throws IOException
324        {
325            // no op
326        }
327
328        @Override
329        public InputStream retrieve( String name )
330            throws IOException, FileNotFoundException
331        {
332            try
333            {
334                log.info( "index update retrieve file, name:{}", name );
335                Path file = tempIndexDirectory.resolve( name );
336                Files.deleteIfExists( file );
337                file.toFile().deleteOnExit();
338                wagon.get( addParameters( name, this.remoteRepository ), file.toFile() );
339                return Files.newInputStream( file );
340            }
341            catch ( AuthorizationException | TransferFailedException e )
342            {
343                throw new IOException( e.getMessage(), e );
344            }
345            catch ( ResourceDoesNotExistException e )
346            {
347                FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
348                fnfe.initCause( e );
349                throw fnfe;
350            }
351        }
352
353        // FIXME remove crappy copy/paste
354        protected String addParameters( String path, RemoteRepository remoteRepository )
355        {
356            if ( remoteRepository.getExtraParameters().isEmpty() )
357            {
358                return path;
359            }
360
361            boolean question = false;
362
363            StringBuilder res = new StringBuilder( path == null ? "" : path );
364
365            for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
366            {
367                if ( !question )
368                {
369                    res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
370                }
371            }
372
373            return res.toString();
374        }
375
376    }
377
378
379}
380