This project has retired. For details please refer to its Attic page.
DownloadRemoteIndexTask xref
View Javadoc
1   package org.apache.archiva.scheduler.indexing.maven;
2   /*
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   */
20  
21  import org.apache.archiva.proxy.maven.WagonFactory;
22  import org.apache.archiva.proxy.maven.WagonFactoryRequest;
23  import org.apache.archiva.proxy.model.NetworkProxy;
24  import org.apache.archiva.repository.base.PasswordCredentials;
25  import org.apache.archiva.repository.RemoteRepository;
26  import org.apache.archiva.repository.RepositoryException;
27  import org.apache.archiva.repository.RepositoryType;
28  import org.apache.archiva.repository.features.RemoteIndexFeature;
29  import org.apache.commons.lang3.time.StopWatch;
30  import org.apache.maven.index.context.IndexingContext;
31  import org.apache.maven.index.updater.IndexUpdateRequest;
32  import org.apache.maven.index.updater.IndexUpdateResult;
33  import org.apache.maven.index.updater.IndexUpdater;
34  import org.apache.maven.index.updater.ResourceFetcher;
35  import org.apache.maven.index_shaded.lucene.index.IndexNotFoundException;
36  import org.apache.maven.wagon.ResourceDoesNotExistException;
37  import org.apache.maven.wagon.StreamWagon;
38  import org.apache.maven.wagon.TransferFailedException;
39  import org.apache.maven.wagon.Wagon;
40  import org.apache.maven.wagon.authentication.AuthenticationInfo;
41  import org.apache.maven.wagon.authorization.AuthorizationException;
42  import org.apache.maven.wagon.events.TransferEvent;
43  import org.apache.maven.wagon.events.TransferListener;
44  import org.apache.maven.wagon.proxy.ProxyInfo;
45  import org.apache.maven.wagon.repository.Repository;
46  import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
47  import org.apache.maven.wagon.shared.http.HttpConfiguration;
48  import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
49  import org.slf4j.Logger;
50  import org.slf4j.LoggerFactory;
51  
52  import java.io.FileNotFoundException;
53  import java.io.IOException;
54  import java.io.InputStream;
55  import java.nio.file.Files;
56  import java.nio.file.Path;
57  import java.nio.file.Paths;
58  import java.util.List;
59  import java.util.Map;
60  
61  /**
62   * @author Olivier Lamy
63   * @since 1.4-M1
64   */
65  public class DownloadRemoteIndexTask
66      implements Runnable
67  {
68      private Logger log = LoggerFactory.getLogger( getClass() );
69  
70      private RemoteRepository remoteRepository;
71  
72      private WagonFactory wagonFactory;
73  
74      private NetworkProxy networkProxy;
75  
76      private boolean fullDownload;
77  
78      private List<String> runningRemoteDownloadIds;
79  
80      private IndexUpdater indexUpdater;
81  
82  
83      public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
84                                      List<String> runningRemoteDownloadIds )
85      {
86          this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
87          this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
88          this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
89          this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
90          this.runningRemoteDownloadIds = runningRemoteDownloadIds;
91          this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
92      }
93  
94      @Override
95      public void run()
96      {
97  
98          // so short lock : not sure we need it
99          synchronized ( this.runningRemoteDownloadIds )
100         {
101             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
102             {
103                 // skip it as it's running
104                 log.info( "skip download index remote for repo {} it's already running",
105                           this.remoteRepository.getId() );
106                 return;
107             }
108             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
109         }
110         Path tempIndexDirectory = null;
111         StopWatch stopWatch = new StopWatch();
112         stopWatch.start();
113         try
114         {
115             log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
116             if (this.remoteRepository.getIndexingContext()==null) {
117                 throw new IndexNotFoundException("No index context set for repository "+remoteRepository.getId());
118             }
119             if (this.remoteRepository.getType()!= RepositoryType.MAVEN) {
120                 throw new RepositoryException("Bad repository type");
121             }
122             if (!this.remoteRepository.supportsFeature(RemoteIndexFeature.class)) {
123                 throw new RepositoryException("Repository does not support RemotIndexFeature "+remoteRepository.getId());
124             }
125             RemoteIndexFeature rif = this.remoteRepository.getFeature(RemoteIndexFeature.class).get();
126             IndexingContext indexingContext = this.remoteRepository.getIndexingContext().getBaseContext(IndexingContext.class);
127             // create a temp directory to download files
128             tempIndexDirectory = Paths.get(indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
129             Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
130             Files.createDirectories( indexCacheDirectory );
131             if ( Files.exists(tempIndexDirectory) )
132             {
133                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
134             }
135             Files.createDirectories( tempIndexDirectory );
136             tempIndexDirectory.toFile().deleteOnExit();
137             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
138 
139             String wagonProtocol = this.remoteRepository.getLocation().getScheme();
140 
141             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
142                 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
143                     this.networkProxy )
144             );
145             // FIXME olamy having 2 config values
146             wagon.setReadTimeout( (int)rif.getDownloadTimeout().toMillis());
147             wagon.setTimeout( (int)remoteRepository.getTimeout().toMillis());
148 
149             if ( wagon instanceof AbstractHttpClientWagon )
150             {
151                 HttpConfiguration httpConfiguration = new HttpConfiguration();
152                 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
153                 httpMethodConfiguration.setUsePreemptive( true );
154                 httpMethodConfiguration.setReadTimeout( (int)rif.getDownloadTimeout().toMillis() );
155                 httpConfiguration.setGet( httpMethodConfiguration );
156                 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
157             }
158 
159             wagon.addTransferListener( new DownloadListener() );
160             ProxyInfo proxyInfo = null;
161             if ( this.networkProxy != null )
162             {
163                 proxyInfo = new ProxyInfo();
164                 proxyInfo.setType( this.networkProxy.getProtocol() );
165                 proxyInfo.setHost( this.networkProxy.getHost() );
166                 proxyInfo.setPort( this.networkProxy.getPort() );
167                 proxyInfo.setUserName( this.networkProxy.getUsername() );
168                 proxyInfo.setPassword( new String(this.networkProxy.getPassword()) );
169             }
170             AuthenticationInfo authenticationInfo = null;
171             if ( this.remoteRepository.getLoginCredentials()!=null && this.remoteRepository.getLoginCredentials() instanceof PasswordCredentials )
172             {
173                 PasswordCredentials creds = (PasswordCredentials) this.remoteRepository.getLoginCredentials();
174                 authenticationInfo = new AuthenticationInfo();
175                 authenticationInfo.setUserName( creds.getUsername());
176                 authenticationInfo.setPassword( new String(creds.getPassword()) );
177             }
178             log.debug("Connection to {}, authInfo={}", this.remoteRepository.getId(), authenticationInfo);
179             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
180                            proxyInfo );
181 
182             Path indexDirectory = indexingContext.getIndexDirectoryFile().toPath();
183             if ( !Files.exists(indexDirectory) )
184             {
185                 Files.createDirectories( indexDirectory );
186             }
187             log.debug("Downloading index file to {}", indexDirectory);
188             log.debug("Index cache dir {}", indexCacheDirectory);
189 
190             ResourceFetcher resourceFetcher =
191                 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
192             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
193             request.setForceFullUpdate( this.fullDownload );
194             request.setLocalIndexCacheDir( indexCacheDirectory.toFile() );
195 
196             IndexUpdateResult result = this.indexUpdater.fetchAndUpdateIndex(request);
197             log.debug("Update result success: {}", result.isSuccessful());
198             stopWatch.stop();
199             log.info( "time update index from remote for repository {}: {}ms", this.remoteRepository.getId(),
200                       ( stopWatch.getTime() ) );
201 
202             // index packing optionnal ??
203             //IndexPackingRequest indexPackingRequest =
204             //    new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
205             //indexPacker.packIndex( indexPackingRequest );
206             indexingContext.updateTimestamp( true );
207 
208         }
209         catch ( Exception e )
210         {
211             log.error( e.getMessage(), e );
212             throw new RuntimeException( e.getMessage(), e );
213         }
214         finally
215         {
216             deleteDirectoryQuiet( tempIndexDirectory );
217             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
218         }
219         log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
220     }
221 
222     private void deleteDirectoryQuiet( Path f )
223     {
224         try
225         {
226             org.apache.archiva.common.utils.FileUtils.deleteDirectory( f );
227         }
228         catch ( IOException e )
229         {
230             log.warn( "skip error delete {} : {}", f, e.getMessage() );
231         }
232     }
233 
234 
235     private static final class DownloadListener
236         implements TransferListener
237     {
238         private Logger log = LoggerFactory.getLogger( getClass() );
239 
240         private String resourceName;
241 
242         private long startTime;
243 
244         private int totalLength = 0;
245 
246         @Override
247         public void transferInitiated( TransferEvent transferEvent )
248         {
249             startTime = System.currentTimeMillis();
250             resourceName = transferEvent.getResource().getName();
251             log.debug( "initiate transfer of {}", resourceName );
252         }
253 
254         @Override
255         public void transferStarted( TransferEvent transferEvent )
256         {
257             this.totalLength = 0;
258             resourceName = transferEvent.getResource().getName();
259             log.info("Transferring: {}, {}",  transferEvent.getResource().getContentLength(), transferEvent.getLocalFile().toString());
260             log.info( "start transfer of {}", transferEvent.getResource().getName() );
261         }
262 
263         @Override
264         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
265         {
266             log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
267             this.totalLength += length;
268         }
269 
270         @Override
271         public void transferCompleted( TransferEvent transferEvent )
272         {
273             resourceName = transferEvent.getResource().getName();
274             long endTime = System.currentTimeMillis();
275             log.info( "end of transfer file {}: {}b, {}ms", transferEvent.getResource().getName(),
276                       this.totalLength, ( endTime - startTime ) );
277         }
278 
279         @Override
280         public void transferError( TransferEvent transferEvent )
281         {
282             log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
283                       transferEvent.getException().getMessage(), transferEvent.getException() );
284         }
285 
286         @Override
287         public void debug( String message )
288         {
289             log.debug( "transfer debug {}", message );
290         }
291     }
292 
293     private static class WagonResourceFetcher
294         implements ResourceFetcher
295     {
296 
297         Logger log;
298 
299         Path tempIndexDirectory;
300 
301         Wagon wagon;
302 
303         RemoteRepository remoteRepository;
304 
305         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
306                                       RemoteRepository remoteRepository )
307         {
308             this.log = log;
309             this.tempIndexDirectory = tempIndexDirectory;
310             this.wagon = wagon;
311             this.remoteRepository = remoteRepository;
312         }
313 
314         @Override
315         public void connect( String id, String url )
316             throws IOException
317         {
318             //no op  
319         }
320 
321         @Override
322         public void disconnect()
323             throws IOException
324         {
325             // no op
326         }
327 
328         @Override
329         public InputStream retrieve( String name )
330             throws IOException, FileNotFoundException
331         {
332             try
333             {
334                 log.info( "index update retrieve file, name:{}", name );
335                 Path file = tempIndexDirectory.resolve( name );
336                 Files.deleteIfExists( file );
337                 file.toFile().deleteOnExit();
338                 wagon.get( addParameters( name, this.remoteRepository ), file.toFile() );
339                 return Files.newInputStream( file );
340             }
341             catch ( AuthorizationException | TransferFailedException e )
342             {
343                 throw new IOException( e.getMessage(), e );
344             }
345             catch ( ResourceDoesNotExistException e )
346             {
347                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
348                 fnfe.initCause( e );
349                 throw fnfe;
350             }
351         }
352 
353         // FIXME remove crappy copy/paste
354         protected String addParameters( String path, RemoteRepository remoteRepository )
355         {
356             if ( remoteRepository.getExtraParameters().isEmpty() )
357             {
358                 return path;
359             }
360 
361             boolean question = false;
362 
363             StringBuilder res = new StringBuilder( path == null ? "" : path );
364 
365             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
366             {
367                 if ( !question )
368                 {
369                     res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
370                 }
371             }
372 
373             return res.toString();
374         }
375 
376     }
377 
378 
379 }
380