Source code

001package org.apache.archiva.repository.scanner;
002
003/*
004 * Licensed to the Apache Software Foundation (ASF) under one
005 * or more contributor license agreements.  See the NOTICE file
006 * distributed with this work for additional information
007 * regarding copyright ownership.  The ASF licenses this file
008 * to you under the Apache License, Version 2.0 (the
009 * "License"); you may not use this file except in compliance
010 * with the License.  You may obtain a copy of the License at
011 *
012 *  http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing,
015 * software distributed under the License is distributed on an
016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
017 * KIND, either express or implied.  See the License for the
018 * specific language governing permissions and limitations
019 * under the License.
020 */
021
022import org.apache.archiva.common.utils.BaseFile;
023import org.apache.archiva.common.utils.PathUtil;
024import org.apache.archiva.consumers.InvalidRepositoryContentConsumer;
025import org.apache.archiva.consumers.KnownRepositoryContentConsumer;
026import org.apache.archiva.consumers.RepositoryContentConsumer;
027import org.apache.archiva.consumers.functors.ConsumerWantsFilePredicate;
028import org.apache.archiva.repository.ManagedRepository;
029import org.apache.archiva.repository.scanner.functors.ConsumerProcessFileClosure;
030import org.apache.archiva.repository.scanner.functors.TriggerBeginScanClosure;
031import org.apache.archiva.repository.scanner.functors.TriggerScanCompletedClosure;
032import org.apache.commons.collections4.Closure;
033import org.apache.commons.collections4.CollectionUtils;
034import org.apache.commons.collections4.IterableUtils;
035import org.apache.commons.collections4.functors.IfClosure;
036import org.apache.commons.lang3.SystemUtils;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040import java.io.IOException;
041import java.nio.file.FileSystem;
042import java.nio.file.FileSystems;
043import java.nio.file.FileVisitResult;
044import java.nio.file.FileVisitor;
045import java.nio.file.Files;
046import java.nio.file.Path;
047import java.nio.file.PathMatcher;
048import java.nio.file.attribute.BasicFileAttributes;
049import java.util.ArrayList;
050import java.util.Date;
051import java.util.HashMap;
052import java.util.List;
053import java.util.Map;
054import java.util.stream.Collectors;
055
056/**
057 * RepositoryScannerInstance
058 */
059public class RepositoryScannerInstance
060    implements FileVisitor<Path>
061{
062    private Logger log = LoggerFactory.getLogger( RepositoryScannerInstance.class );
063
064    /**
065     * Consumers that process known content.
066     */
067    private List<KnownRepositoryContentConsumer> knownConsumers;
068
069    /**
070     * Consumers that process unknown/invalid content.
071     */
072    private List<InvalidRepositoryContentConsumer> invalidConsumers;
073
074    private ManagedRepository repository;
075
076    private RepositoryScanStatistics stats;
077
078    private long changesSince = 0;
079
080    private ConsumerProcessFileClosure consumerProcessFile;
081
082    private ConsumerWantsFilePredicate consumerWantsFile;
083
084    private Map<String, Long> consumerTimings;
085
086    private Map<String, Long> consumerCounts;
087
088
089    private List<String> fileNameIncludePattern = new ArrayList<>();
090    private List<String> fileNameExcludePattern = new ArrayList<>();
091
092    private List<PathMatcher> includeMatcher = new ArrayList<>();
093    private List<PathMatcher> excludeMatcher = new ArrayList<>();
094
095    private boolean isRunning = false;
096
097    Path basePath = null;
098
099    public RepositoryScannerInstance( ManagedRepository repository,
100                                      List<KnownRepositoryContentConsumer> knownConsumerList,
101                                      List<InvalidRepositoryContentConsumer> invalidConsumerList )
102    {
103        this.repository = repository;
104        this.knownConsumers = knownConsumerList;
105        this.invalidConsumers = invalidConsumerList;
106
107        addFileNameIncludePattern("**/*");
108
109        consumerTimings = new HashMap<>();
110        consumerCounts = new HashMap<>();
111
112        this.consumerProcessFile = new ConsumerProcessFileClosure();
113        consumerProcessFile.setExecuteOnEntireRepo( true );
114        consumerProcessFile.setConsumerTimings( consumerTimings );
115        consumerProcessFile.setConsumerCounts( consumerCounts );
116
117        this.consumerWantsFile = new ConsumerWantsFilePredicate( repository );
118
119        stats = new RepositoryScanStatistics();
120        stats.setRepositoryId( repository.getId() );
121
122        Closure<RepositoryContentConsumer> triggerBeginScan =
123            new TriggerBeginScanClosure( repository, new Date( System.currentTimeMillis() ), true );
124
125        IterableUtils.forEach( knownConsumerList, triggerBeginScan );
126        IterableUtils.forEach( invalidConsumerList, triggerBeginScan );
127
128        if ( SystemUtils.IS_OS_WINDOWS )
129        {
130            consumerWantsFile.setCaseSensitive( false );
131        }
132    }
133
134    public RepositoryScannerInstance( ManagedRepository repository,
135                                      List<KnownRepositoryContentConsumer> knownContentConsumers,
136                                      List<InvalidRepositoryContentConsumer> invalidContentConsumers,
137                                      long changesSince )
138    {
139        this( repository, knownContentConsumers, invalidContentConsumers );
140
141        consumerWantsFile.setChangesSince( changesSince );
142
143        this.changesSince = changesSince;
144    }
145
146    public RepositoryScanStatistics getStatistics()
147    {
148        return stats;
149    }
150
151    public Map<String, Long> getConsumerTimings()
152    {
153        return consumerTimings;
154    }
155
156    public Map<String, Long> getConsumerCounts()
157    {
158        return consumerCounts;
159    }
160
161    public ManagedRepository getRepository()
162    {
163        return repository;
164    }
165
166    public RepositoryScanStatistics getStats()
167    {
168        return stats;
169    }
170
171    public long getChangesSince()
172    {
173        return changesSince;
174    }
175
176    public List<String> getFileNameIncludePattern() {
177        return fileNameIncludePattern;
178    }
179
180    public void setFileNameIncludePattern(List<String> fileNamePattern) {
181        this.fileNameIncludePattern = fileNamePattern;
182        FileSystem sys = FileSystems.getDefault();
183        this.includeMatcher = fileNamePattern.stream().map(ts ->sys
184                .getPathMatcher("glob:" + ts)).collect(Collectors.toList());
185    }
186
187    public void addFileNameIncludePattern(String fileNamePattern) {
188        if (! this.fileNameIncludePattern.contains(fileNamePattern)) {
189            this.fileNameIncludePattern.add(fileNamePattern);
190            this.includeMatcher.add(FileSystems.getDefault().getPathMatcher("glob:" + fileNamePattern));
191        }
192    }
193
194    public List<String> getFileNameExcludePattern() {
195        return fileNameExcludePattern;
196    }
197
198    public void setFileNameExcludePattern(List<String> fileNamePattern) {
199        this.fileNameExcludePattern = fileNamePattern;
200        FileSystem sys = FileSystems.getDefault();
201        this.excludeMatcher = fileNamePattern.stream().map(ts ->sys
202                .getPathMatcher("glob:" + ts)).collect(Collectors.toList());
203    }
204
205    public void addFileNameExcludePattern(String fileNamePattern) {
206        if (! this.fileNameExcludePattern.contains(fileNamePattern)) {
207            this.fileNameExcludePattern.add(fileNamePattern);
208            this.excludeMatcher.add(FileSystems.getDefault().getPathMatcher("glob:" + fileNamePattern));
209        }
210    }
211
212
213    @Override
214    public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
215        if (!isRunning) {
216            isRunning = true;
217            this.basePath = dir;
218            log.info( "Walk Started: [{}] {}", this.repository.getId(), this.repository.getLocation() );
219            stats.triggerStart();
220        }
221        return FileVisitResult.CONTINUE;
222    }
223
224    @Override
225    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
226        final Path relativeFile = basePath.relativize( file );
227        if (excludeMatcher.stream().noneMatch(m -> m.matches(relativeFile)) && includeMatcher.stream().allMatch(m -> m.matches(relativeFile))) {
228            log.debug( "Walk Step: {}, {}", file );
229
230            stats.increaseFileCount();
231
232            // consume files regardless - the predicate will check the timestamp
233            Path repoPath = PathUtil.getPathFromUri( repository.getLocation() );
234            BaseFile basefile = new BaseFile( repoPath.toString(), file.toFile() );
235
236            // Timestamp finished points to the last successful scan, not this current one.
237            if ( Files.getLastModifiedTime(file).toMillis() >= changesSince )
238            {
239                stats.increaseNewFileCount();
240            }
241
242            consumerProcessFile.setBasefile( basefile );
243            consumerWantsFile.setBasefile( basefile );
244
245            Closure<RepositoryContentConsumer> processIfWanted = IfClosure.ifClosure( consumerWantsFile, consumerProcessFile );
246            IterableUtils.forEach( this.knownConsumers, processIfWanted );
247
248            if ( consumerWantsFile.getWantedFileCount() <= 0 )
249            {
250                // Nothing known processed this file.  It is invalid!
251                IterableUtils.forEach( this.invalidConsumers, consumerProcessFile );
252            }
253
254        }
255        return FileVisitResult.CONTINUE;
256    }
257
258    @Override
259    public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
260        log.error("Error occured at {}: {}", file, exc.getMessage(), exc);
261        try
262        {
263            if ( basePath != null && Files.isSameFile( file, basePath ) )
264            {
265                log.debug( "Finishing walk from visitFileFailed" );
266                finishWalk( );
267            }
268        } catch (Throwable e) {
269            log.error( "Error during visitFileFailed handling: {}", e.getMessage( ), e );
270        }
271        return FileVisitResult.CONTINUE;
272    }
273
274    @Override
275    public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
276        if (Files.isSameFile(dir, basePath)) {
277            finishWalk();
278        }
279        return FileVisitResult.CONTINUE;
280    }
281
282    private void finishWalk() {
283        this.isRunning = false;
284        TriggerScanCompletedClosure scanCompletedClosure = new TriggerScanCompletedClosure( repository, true );
285        IterableUtils.forEach( knownConsumers, scanCompletedClosure );
286        IterableUtils.forEach( invalidConsumers, scanCompletedClosure );
287
288        stats.setConsumerTimings( consumerTimings );
289        stats.setConsumerCounts( consumerCounts );
290
291        log.info( "Walk Finished: [{}] {}", this.repository.getId(), this.repository.getLocation() );
292        stats.triggerFinished();
293        this.basePath = null;
294    }
295}