This project has retired. For details please refer to its
Attic page.
DownloadRemoteIndexTask xref
1 package org.apache.archiva.scheduler.indexing;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 import org.apache.archiva.admin.model.RepositoryAdminException;
22 import org.apache.archiva.admin.model.beans.NetworkProxy;
23 import org.apache.archiva.admin.model.beans.RemoteRepository;
24 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
25 import org.apache.archiva.proxy.common.WagonFactory;
26 import org.apache.archiva.proxy.common.WagonFactoryException;
27 import org.apache.archiva.proxy.common.WagonFactoryRequest;
28 import org.apache.commons.io.FileUtils;
29 import org.apache.commons.lang.time.StopWatch;
30 import org.apache.maven.index.context.IndexingContext;
31 import org.apache.maven.index.updater.IndexUpdateRequest;
32 import org.apache.maven.index.updater.IndexUpdater;
33 import org.apache.maven.index.updater.ResourceFetcher;
34 import org.apache.maven.wagon.ConnectionException;
35 import org.apache.maven.wagon.ResourceDoesNotExistException;
36 import org.apache.maven.wagon.StreamWagon;
37 import org.apache.maven.wagon.TransferFailedException;
38 import org.apache.maven.wagon.Wagon;
39 import org.apache.maven.wagon.authentication.AuthenticationException;
40 import org.apache.maven.wagon.authentication.AuthenticationInfo;
41 import org.apache.maven.wagon.authorization.AuthorizationException;
42 import org.apache.maven.wagon.events.TransferEvent;
43 import org.apache.maven.wagon.events.TransferListener;
44 import org.apache.maven.wagon.providers.http.AbstractHttpClientWagon;
45 import org.apache.maven.wagon.providers.http.HttpConfiguration;
46 import org.apache.maven.wagon.providers.http.HttpMethodConfiguration;
47 import org.apache.maven.wagon.proxy.ProxyInfo;
48 import org.apache.maven.wagon.repository.Repository;
49 import org.slf4j.Logger;
50 import org.slf4j.LoggerFactory;
51
52 import java.io.File;
53 import java.io.FileNotFoundException;
54 import java.io.IOException;
55 import java.io.InputStream;
56 import java.net.MalformedURLException;
57 import java.net.URL;
58 import java.nio.file.Files;
59 import java.util.List;
60 import java.util.Map;
61
62
63
64
65
66 public class DownloadRemoteIndexTask
67 implements Runnable
68 {
69 private Logger log = LoggerFactory.getLogger( getClass() );
70
71 private RemoteRepository remoteRepository;
72
73 private RemoteRepositoryAdmin remoteRepositoryAdmin;
74
75 private WagonFactory wagonFactory;
76
77 private NetworkProxy networkProxy;
78
79 private boolean fullDownload;
80
81 private List<String> runningRemoteDownloadIds;
82
83 private IndexUpdater indexUpdater;
84
85
86 public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
87 List<String> runningRemoteDownloadIds )
88 {
89 this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
90 this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
91 this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
92 this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
93 this.runningRemoteDownloadIds = runningRemoteDownloadIds;
94 this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
95 this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
96 }
97
98 @Override
99 public void run()
100 {
101
102
103 synchronized ( this.runningRemoteDownloadIds )
104 {
105 if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
106 {
107
108 log.info( "skip download index remote for repo {} it's already running",
109 this.remoteRepository.getId() );
110 return;
111 }
112 this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
113 }
114 File tempIndexDirectory = null;
115 StopWatch stopWatch = new StopWatch();
116 stopWatch.start();
117 try
118 {
119 log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
120 IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
121
122
123 tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
124 File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
125 indexCacheDirectory.mkdirs();
126 if ( tempIndexDirectory.exists() )
127 {
128 FileUtils.deleteDirectory( tempIndexDirectory );
129 }
130 tempIndexDirectory.mkdirs();
131 tempIndexDirectory.deleteOnExit();
132 String baseIndexUrl = indexingContext.getIndexUpdateUrl();
133
134 String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol();
135
136 final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
137 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
138 this.networkProxy )
139 );
140
141 wagon.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
142 wagon.setTimeout( remoteRepository.getTimeout() * 1000 );
143
144 if ( wagon instanceof AbstractHttpClientWagon )
145 {
146 HttpConfiguration httpConfiguration = new HttpConfiguration();
147 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
148 httpMethodConfiguration.setUsePreemptive( true );
149 httpMethodConfiguration.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
150 httpConfiguration.setGet( httpMethodConfiguration );
151 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
152 }
153
154 wagon.addTransferListener( new DownloadListener() );
155 ProxyInfo proxyInfo = null;
156 if ( this.networkProxy != null )
157 {
158 proxyInfo = new ProxyInfo();
159 proxyInfo.setType( this.networkProxy.getProtocol() );
160 proxyInfo.setHost( this.networkProxy.getHost() );
161 proxyInfo.setPort( this.networkProxy.getPort() );
162 proxyInfo.setUserName( this.networkProxy.getUsername() );
163 proxyInfo.setPassword( this.networkProxy.getPassword() );
164 }
165 AuthenticationInfo authenticationInfo = null;
166 if ( this.remoteRepository.getUserName() != null )
167 {
168 authenticationInfo = new AuthenticationInfo();
169 authenticationInfo.setUserName( this.remoteRepository.getUserName() );
170 authenticationInfo.setPassword( this.remoteRepository.getPassword() );
171 }
172 wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
173 proxyInfo );
174
175 File indexDirectory = indexingContext.getIndexDirectoryFile();
176 if ( !indexDirectory.exists() )
177 {
178 indexDirectory.mkdirs();
179 }
180
181 ResourceFetcher resourceFetcher =
182 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
183 IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
184 request.setForceFullUpdate( this.fullDownload );
185 request.setLocalIndexCacheDir( indexCacheDirectory );
186
187 this.indexUpdater.fetchAndUpdateIndex( request );
188 stopWatch.stop();
189 log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(),
190 ( stopWatch.getTime() / 1000 ) );
191
192
193
194
195
196 indexingContext.updateTimestamp( true );
197
198 }
199 catch ( MalformedURLException e )
200 {
201 log.error( e.getMessage(), e );
202 throw new RuntimeException( e.getMessage(), e );
203 }
204 catch ( WagonFactoryException e )
205 {
206 log.error( e.getMessage(), e );
207 throw new RuntimeException( e.getMessage(), e );
208 }
209 catch ( ConnectionException e )
210 {
211 log.error( e.getMessage(), e );
212 throw new RuntimeException( e.getMessage(), e );
213 }
214 catch ( AuthenticationException e )
215 {
216 log.error( e.getMessage(), e );
217 throw new RuntimeException( e.getMessage(), e );
218 }
219 catch ( IOException e )
220 {
221 log.error( e.getMessage(), e );
222 throw new RuntimeException( e.getMessage(), e );
223 }
224 catch ( RepositoryAdminException e )
225 {
226 log.error( e.getMessage(), e );
227 throw new RuntimeException( e.getMessage(), e );
228 }
229 finally
230 {
231 deleteDirectoryQuiet( tempIndexDirectory );
232 this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
233 }
234 log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
235 }
236
237 private void deleteDirectoryQuiet( File f )
238 {
239 try
240 {
241 FileUtils.deleteDirectory( f );
242 }
243 catch ( IOException e )
244 {
245 log.warn( "skip error delete {} : {}", f, e.getMessage() );
246 }
247 }
248
249
250 private static final class DownloadListener
251 implements TransferListener
252 {
253 private Logger log = LoggerFactory.getLogger( getClass() );
254
255 private String resourceName;
256
257 private long startTime;
258
259 private int totalLength = 0;
260
261 @Override
262 public void transferInitiated( TransferEvent transferEvent )
263 {
264 startTime = System.currentTimeMillis();
265 resourceName = transferEvent.getResource().getName();
266 log.debug( "initiate transfer of {}", resourceName );
267 }
268
269 @Override
270 public void transferStarted( TransferEvent transferEvent )
271 {
272 this.totalLength = 0;
273 resourceName = transferEvent.getResource().getName();
274 log.info( "start transfer of {}", transferEvent.getResource().getName() );
275 }
276
277 @Override
278 public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
279 {
280 log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
281 this.totalLength += length;
282 }
283
284 @Override
285 public void transferCompleted( TransferEvent transferEvent )
286 {
287 resourceName = transferEvent.getResource().getName();
288 long endTime = System.currentTimeMillis();
289 log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(),
290 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
291 }
292
293 @Override
294 public void transferError( TransferEvent transferEvent )
295 {
296 log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
297 transferEvent.getException().getMessage(), transferEvent.getException() );
298 }
299
300 @Override
301 public void debug( String message )
302 {
303 log.debug( "transfer debug {}", message );
304 }
305 }
306
307 private static class WagonResourceFetcher
308 implements ResourceFetcher
309 {
310
311 Logger log;
312
313 File tempIndexDirectory;
314
315 Wagon wagon;
316
317 RemoteRepository remoteRepository;
318
319 private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon,
320 RemoteRepository remoteRepository )
321 {
322 this.log = log;
323 this.tempIndexDirectory = tempIndexDirectory;
324 this.wagon = wagon;
325 this.remoteRepository = remoteRepository;
326 }
327
328 @Override
329 public void connect( String id, String url )
330 throws IOException
331 {
332
333 }
334
335 @Override
336 public void disconnect()
337 throws IOException
338 {
339
340 }
341
342 @Override
343 public InputStream retrieve( String name )
344 throws IOException, FileNotFoundException
345 {
346 try
347 {
348 log.info( "index update retrieve file, name:{}", name );
349 File file = new File( tempIndexDirectory, name );
350 Files.deleteIfExists( file.toPath() );
351 file.deleteOnExit();
352 wagon.get( addParameters( name, this.remoteRepository ), file );
353 return Files.newInputStream( file.toPath() );
354 }
355 catch ( AuthorizationException | TransferFailedException e )
356 {
357 throw new IOException( e.getMessage(), e );
358 }
359 catch ( ResourceDoesNotExistException e )
360 {
361 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
362 fnfe.initCause( e );
363 throw fnfe;
364 }
365 }
366
367
368 protected String addParameters( String path, RemoteRepository remoteRepository )
369 {
370 if ( remoteRepository.getExtraParameters().isEmpty() )
371 {
372 return path;
373 }
374
375 boolean question = false;
376
377 StringBuilder res = new StringBuilder( path == null ? "" : path );
378
379 for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
380 {
381 if ( !question )
382 {
383 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
384 }
385 }
386
387 return res.toString();
388 }
389
390 }
391
392
393 }
394