This project has retired. For details please refer to its Attic page.
DownloadRemoteIndexTask xref
View Javadoc
1   package org.apache.archiva.scheduler.indexing;
2   /*
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   */
20  
21  import org.apache.archiva.admin.model.RepositoryAdminException;
22  import org.apache.archiva.admin.model.beans.NetworkProxy;
23  import org.apache.archiva.admin.model.beans.RemoteRepository;
24  import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
25  import org.apache.archiva.proxy.common.WagonFactory;
26  import org.apache.archiva.proxy.common.WagonFactoryException;
27  import org.apache.archiva.proxy.common.WagonFactoryRequest;
28  import org.apache.commons.io.FileUtils;
29  import org.apache.commons.lang.time.StopWatch;
30  import org.apache.maven.index.context.IndexingContext;
31  import org.apache.maven.index.updater.IndexUpdateRequest;
32  import org.apache.maven.index.updater.IndexUpdater;
33  import org.apache.maven.index.updater.ResourceFetcher;
34  import org.apache.maven.wagon.ConnectionException;
35  import org.apache.maven.wagon.ResourceDoesNotExistException;
36  import org.apache.maven.wagon.StreamWagon;
37  import org.apache.maven.wagon.TransferFailedException;
38  import org.apache.maven.wagon.Wagon;
39  import org.apache.maven.wagon.authentication.AuthenticationException;
40  import org.apache.maven.wagon.authentication.AuthenticationInfo;
41  import org.apache.maven.wagon.authorization.AuthorizationException;
42  import org.apache.maven.wagon.events.TransferEvent;
43  import org.apache.maven.wagon.events.TransferListener;
44  import org.apache.maven.wagon.providers.http.AbstractHttpClientWagon;
45  import org.apache.maven.wagon.providers.http.HttpConfiguration;
46  import org.apache.maven.wagon.providers.http.HttpMethodConfiguration;
47  import org.apache.maven.wagon.proxy.ProxyInfo;
48  import org.apache.maven.wagon.repository.Repository;
49  import org.slf4j.Logger;
50  import org.slf4j.LoggerFactory;
51  
52  import java.io.File;
53  import java.io.FileNotFoundException;
54  import java.io.IOException;
55  import java.io.InputStream;
56  import java.net.MalformedURLException;
57  import java.net.URL;
58  import java.nio.file.Files;
59  import java.util.List;
60  import java.util.Map;
61  
62  /**
63   * @author Olivier Lamy
64   * @since 1.4-M1
65   */
66  public class DownloadRemoteIndexTask
67      implements Runnable
68  {
69      private Logger log = LoggerFactory.getLogger( getClass() );
70  
71      private RemoteRepository remoteRepository;
72  
73      private RemoteRepositoryAdmin remoteRepositoryAdmin;
74  
75      private WagonFactory wagonFactory;
76  
77      private NetworkProxy networkProxy;
78  
79      private boolean fullDownload;
80  
81      private List<String> runningRemoteDownloadIds;
82  
83      private IndexUpdater indexUpdater;
84  
85  
86      public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
87                                      List<String> runningRemoteDownloadIds )
88      {
89          this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
90          this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
91          this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
92          this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
93          this.runningRemoteDownloadIds = runningRemoteDownloadIds;
94          this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
95          this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
96      }
97  
98      @Override
99      public void run()
100     {
101 
102         // so short lock : not sure we need it
103         synchronized ( this.runningRemoteDownloadIds )
104         {
105             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
106             {
107                 // skip it as it's running
108                 log.info( "skip download index remote for repo {} it's already running",
109                           this.remoteRepository.getId() );
110                 return;
111             }
112             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
113         }
114         File tempIndexDirectory = null;
115         StopWatch stopWatch = new StopWatch();
116         stopWatch.start();
117         try
118         {
119             log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
120             IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
121 
122             // create a temp directory to download files
123             tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
124             File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
125             indexCacheDirectory.mkdirs();
126             if ( tempIndexDirectory.exists() )
127             {
128                 FileUtils.deleteDirectory( tempIndexDirectory );
129             }
130             tempIndexDirectory.mkdirs();
131             tempIndexDirectory.deleteOnExit();
132             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
133 
134             String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol();
135 
136             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
137                 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
138                     this.networkProxy )
139             );
140             // FIXME olamy having 2 config values
141             wagon.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
142             wagon.setTimeout( remoteRepository.getTimeout() * 1000 );
143 
144             if ( wagon instanceof AbstractHttpClientWagon )
145             {
146                 HttpConfiguration httpConfiguration = new HttpConfiguration();
147                 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
148                 httpMethodConfiguration.setUsePreemptive( true );
149                 httpMethodConfiguration.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
150                 httpConfiguration.setGet( httpMethodConfiguration );
151                 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
152             }
153 
154             wagon.addTransferListener( new DownloadListener() );
155             ProxyInfo proxyInfo = null;
156             if ( this.networkProxy != null )
157             {
158                 proxyInfo = new ProxyInfo();
159                 proxyInfo.setType( this.networkProxy.getProtocol() );
160                 proxyInfo.setHost( this.networkProxy.getHost() );
161                 proxyInfo.setPort( this.networkProxy.getPort() );
162                 proxyInfo.setUserName( this.networkProxy.getUsername() );
163                 proxyInfo.setPassword( this.networkProxy.getPassword() );
164             }
165             AuthenticationInfo authenticationInfo = null;
166             if ( this.remoteRepository.getUserName() != null )
167             {
168                 authenticationInfo = new AuthenticationInfo();
169                 authenticationInfo.setUserName( this.remoteRepository.getUserName() );
170                 authenticationInfo.setPassword( this.remoteRepository.getPassword() );
171             }
172             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
173                            proxyInfo );
174 
175             File indexDirectory = indexingContext.getIndexDirectoryFile();
176             if ( !indexDirectory.exists() )
177             {
178                 indexDirectory.mkdirs();
179             }
180 
181             ResourceFetcher resourceFetcher =
182                 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
183             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
184             request.setForceFullUpdate( this.fullDownload );
185             request.setLocalIndexCacheDir( indexCacheDirectory );
186 
187             this.indexUpdater.fetchAndUpdateIndex( request );
188             stopWatch.stop();
189             log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(),
190                       ( stopWatch.getTime() / 1000 ) );
191 
192             // index packing optionnal ??
193             //IndexPackingRequest indexPackingRequest =
194             //    new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
195             //indexPacker.packIndex( indexPackingRequest );
196             indexingContext.updateTimestamp( true );
197 
198         }
199         catch ( MalformedURLException e )
200         {
201             log.error( e.getMessage(), e );
202             throw new RuntimeException( e.getMessage(), e );
203         }
204         catch ( WagonFactoryException e )
205         {
206             log.error( e.getMessage(), e );
207             throw new RuntimeException( e.getMessage(), e );
208         }
209         catch ( ConnectionException e )
210         {
211             log.error( e.getMessage(), e );
212             throw new RuntimeException( e.getMessage(), e );
213         }
214         catch ( AuthenticationException e )
215         {
216             log.error( e.getMessage(), e );
217             throw new RuntimeException( e.getMessage(), e );
218         }
219         catch ( IOException e )
220         {
221             log.error( e.getMessage(), e );
222             throw new RuntimeException( e.getMessage(), e );
223         }
224         catch ( RepositoryAdminException e )
225         {
226             log.error( e.getMessage(), e );
227             throw new RuntimeException( e.getMessage(), e );
228         }
229         finally
230         {
231             deleteDirectoryQuiet( tempIndexDirectory );
232             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
233         }
234         log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
235     }
236 
237     private void deleteDirectoryQuiet( File f )
238     {
239         try
240         {
241             FileUtils.deleteDirectory( f );
242         }
243         catch ( IOException e )
244         {
245             log.warn( "skip error delete {} : {}", f, e.getMessage() );
246         }
247     }
248 
249 
250     private static final class DownloadListener
251         implements TransferListener
252     {
253         private Logger log = LoggerFactory.getLogger( getClass() );
254 
255         private String resourceName;
256 
257         private long startTime;
258 
259         private int totalLength = 0;
260 
261         @Override
262         public void transferInitiated( TransferEvent transferEvent )
263         {
264             startTime = System.currentTimeMillis();
265             resourceName = transferEvent.getResource().getName();
266             log.debug( "initiate transfer of {}", resourceName );
267         }
268 
269         @Override
270         public void transferStarted( TransferEvent transferEvent )
271         {
272             this.totalLength = 0;
273             resourceName = transferEvent.getResource().getName();
274             log.info( "start transfer of {}", transferEvent.getResource().getName() );
275         }
276 
277         @Override
278         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
279         {
280             log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
281             this.totalLength += length;
282         }
283 
284         @Override
285         public void transferCompleted( TransferEvent transferEvent )
286         {
287             resourceName = transferEvent.getResource().getName();
288             long endTime = System.currentTimeMillis();
289             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(),
290                       this.totalLength / 1024, ( endTime - startTime ) / 1000 );
291         }
292 
293         @Override
294         public void transferError( TransferEvent transferEvent )
295         {
296             log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
297                       transferEvent.getException().getMessage(), transferEvent.getException() );
298         }
299 
300         @Override
301         public void debug( String message )
302         {
303             log.debug( "transfer debug {}", message );
304         }
305     }
306 
307     private static class WagonResourceFetcher
308         implements ResourceFetcher
309     {
310 
311         Logger log;
312 
313         File tempIndexDirectory;
314 
315         Wagon wagon;
316 
317         RemoteRepository remoteRepository;
318 
319         private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon,
320                                       RemoteRepository remoteRepository )
321         {
322             this.log = log;
323             this.tempIndexDirectory = tempIndexDirectory;
324             this.wagon = wagon;
325             this.remoteRepository = remoteRepository;
326         }
327 
328         @Override
329         public void connect( String id, String url )
330             throws IOException
331         {
332             //no op  
333         }
334 
335         @Override
336         public void disconnect()
337             throws IOException
338         {
339             // no op
340         }
341 
342         @Override
343         public InputStream retrieve( String name )
344             throws IOException, FileNotFoundException
345         {
346             try
347             {
348                 log.info( "index update retrieve file, name:{}", name );
349                 File file = new File( tempIndexDirectory, name );
350                 Files.deleteIfExists( file.toPath() );
351                 file.deleteOnExit();
352                 wagon.get( addParameters( name, this.remoteRepository ), file );
353                 return Files.newInputStream( file.toPath() );
354             }
355             catch ( AuthorizationException | TransferFailedException e )
356             {
357                 throw new IOException( e.getMessage(), e );
358             }
359             catch ( ResourceDoesNotExistException e )
360             {
361                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
362                 fnfe.initCause( e );
363                 throw fnfe;
364             }
365         }
366 
367         // FIXME remove crappy copy/paste
368         protected String addParameters( String path, RemoteRepository remoteRepository )
369         {
370             if ( remoteRepository.getExtraParameters().isEmpty() )
371             {
372                 return path;
373             }
374 
375             boolean question = false;
376 
377             StringBuilder res = new StringBuilder( path == null ? "" : path );
378 
379             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
380             {
381                 if ( !question )
382                 {
383                     res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
384                 }
385             }
386 
387             return res.toString();
388         }
389 
390     }
391 
392 
393 }
394