queue) {
this.queue = queue;
- this.scheduler = new JobScheduler(this);
- }
-
- @Override
- public final Scheduler getScheduler() {
- return scheduler;
}
@Nonnull
diff --git a/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractPriorityJobQueue.java
similarity index 90%
rename from src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
rename to src/main/java/ai/preferred/venom/job/AbstractPriorityJobQueue.java
index 41c1fc3..1d64ca1 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractPriorityJobQueue.java
@@ -24,12 +24,12 @@
/**
* @author Ween Jiann Lee
*/
-public abstract class AbstractPriorityQueueScheduler extends AbstractQueueScheduler {
+public abstract class AbstractPriorityJobQueue extends AbstractJobQueue {
/**
- * Constructs an instance of AbstractQueueScheduler.
+ * Constructs an instance of AbstractJobQueue.
*/
- protected AbstractPriorityQueueScheduler() {
+ protected AbstractPriorityJobQueue() {
super(new PriorityBlockingQueue<>(11,
Comparator.comparing(o -> (o.getJobAttribute(PriorityJobAttribute.class)))));
}
diff --git a/src/main/java/ai/preferred/venom/job/FIFOJobQueue.java b/src/main/java/ai/preferred/venom/job/FIFOJobQueue.java
new file mode 100644
index 0000000..fbc6e29
--- /dev/null
+++ b/src/main/java/ai/preferred/venom/job/FIFOJobQueue.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2018 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+import javax.annotation.Nonnull;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This class provides and implementation of scheduler with a first in
+ * first out queue.
+ *
+ * Jobs in queue will be processed first in order of insertion.
+ *
+ *
+ * @author Ween Jiann Lee
+ */
+public class FIFOJobQueue extends AbstractJobQueue {
+
+ /**
+ * Constructs an instance of FIFOJobQueue.
+ */
+ public FIFOJobQueue() {
+ super(new LinkedBlockingQueue<>());
+ }
+
+ @Override
+ public final void put(final @Nonnull Job job) throws InterruptedException {
+ getQueue().put(job);
+ }
+
+ @Override
+ public final boolean offer(final Job job, final long timeout, final @Nonnull TimeUnit unit)
+ throws InterruptedException {
+ return getQueue().offer(job, timeout, unit);
+ }
+
+ @Override
+ public final boolean offer(final @Nonnull Job job) {
+ return getQueue().offer(job);
+ }
+
+ @Override
+ public final Job poll(final long timeout, final @Nonnull TimeUnit unit) throws InterruptedException {
+ return getQueue().poll(timeout, unit);
+ }
+
+ @Override
+ public final Job poll() {
+ return getQueue().poll();
+ }
+}
diff --git a/src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java b/src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java
index db9d411..58cfd1a 100644
--- a/src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java
@@ -1,66 +1,9 @@
-/*
- * Copyright 2018 Preferred.AI
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
package ai.preferred.venom.job;
-import javax.annotation.Nonnull;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
/**
- * This class provides and implementation of scheduler with a first in
- * first out queue.
- *
- * Jobs in queue will be processed first in order of insertion.
- *
- *
- * @author Ween Jiann Lee
+ * For backwards compatibility.
*/
-public class FIFOQueueScheduler extends AbstractQueueScheduler {
-
- /**
- * Constructs an instance of FIFOQueueScheduler.
- */
- public FIFOQueueScheduler() {
- super(new LinkedBlockingQueue<>());
- }
-
- @Override
- public final void put(final @Nonnull Job job) throws InterruptedException {
- getQueue().put(job);
- }
-
- @Override
- public final boolean offer(final Job job, final long timeout, final @Nonnull TimeUnit unit)
- throws InterruptedException {
- return getQueue().offer(job, timeout, unit);
- }
-
- @Override
- public final boolean offer(final @Nonnull Job job) {
- return getQueue().offer(job);
- }
-
- @Override
- public final Job poll(final long timeout, final @Nonnull TimeUnit unit) throws InterruptedException {
- return getQueue().poll(timeout, unit);
- }
+@Deprecated
+public class FIFOQueueScheduler extends FIFOJobQueue {
- @Override
- public final Job poll() {
- return getQueue().poll();
- }
}
diff --git a/src/main/java/ai/preferred/venom/job/JobScheduler.java b/src/main/java/ai/preferred/venom/job/JobScheduler.java
deleted file mode 100644
index 537ccff..0000000
--- a/src/main/java/ai/preferred/venom/job/JobScheduler.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright (c) 2019 Preferred.AI
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ai.preferred.venom.job;
-
-import ai.preferred.venom.Handler;
-import ai.preferred.venom.request.Request;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.validation.constraints.NotNull;
-
-/**
- * An implementation of ai.preferred.venom.job.Scheduler using Job.
- */
-public class JobScheduler implements Scheduler {
-
- /**
- * Logger.
- */
- private static final Logger LOGGER = LoggerFactory.getLogger(JobScheduler.class);
-
- /**
- * The queue used for this scheduler.
- */
- private final QueueScheduler queueScheduler;
-
- /**
- * Constructs an instance of JobScheduler.
- *
- * @param queueScheduler an instance of BlockingQueue
- */
- public JobScheduler(final QueueScheduler queueScheduler) {
- this.queueScheduler = queueScheduler;
- }
-
- @Override
- public final void add(final @NotNull Request request, final Handler handler,
- final @NotNull JobAttribute... jobAttributes) {
- final Job job = new Job(request, handler, jobAttributes);
- queueScheduler.add(job);
- LOGGER.debug("Job {} - {} added to queue.", job.toString(), request.getUrl());
- }
-
- @Override
- public final void add(final @NotNull Request request, final @NotNull JobAttribute... jobAttributes) {
- add(request, null, jobAttributes);
- }
-
- @Override
- public final void add(final Request request, final @NotNull Handler handler) {
- add(request, handler, new JobAttribute[0]);
- }
-
- @Override
- public final void add(final @NotNull Request request) {
- add(request, null, new JobAttribute[0]);
- }
-
- @Override
- public final void add(final @NotNull Request r, final Handler h, final Priority p, final Priority pf) {
- add(r, h, new PriorityJobAttribute(p, pf));
- }
-
- @Override
- public final void add(final @NotNull Request r, final Handler h, final Priority p) {
- add(r, h, p, Priority.FLOOR);
- }
-
- @Override
- public final void add(final @NotNull Request r, final Priority p, final Priority pf) {
- add(r, null, p, pf);
- }
-
- @Override
- public final void add(final @NotNull Request r, final Priority p) {
- add(r, (Handler) null, p);
- }
-}
diff --git a/src/main/java/ai/preferred/venom/job/LazyPriorityQueueScheduler.java b/src/main/java/ai/preferred/venom/job/LazyPriorityJobQueue.java
similarity index 91%
rename from src/main/java/ai/preferred/venom/job/LazyPriorityQueueScheduler.java
rename to src/main/java/ai/preferred/venom/job/LazyPriorityJobQueue.java
index a496656..19d31b6 100644
--- a/src/main/java/ai/preferred/venom/job/LazyPriorityQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/LazyPriorityJobQueue.java
@@ -35,7 +35,7 @@
* @author Maksim Tkachenko
* @author Ween Jiann Lee
*/
-public class LazyPriorityQueueScheduler extends AbstractPriorityQueueScheduler {
+public class LazyPriorityJobQueue extends AbstractPriorityJobQueue {
/**
* An object to synchronise upon.
@@ -58,7 +58,7 @@ public class LazyPriorityQueueScheduler extends AbstractPriorityQueueScheduler {
* @param requests An iterator to obtain requests
* @param handler The default handler to use
*/
- public LazyPriorityQueueScheduler(final Iterator requests, final Handler handler) {
+ public LazyPriorityJobQueue(final Iterator requests, final Handler handler) {
this.requests = requests;
this.handler = handler;
}
@@ -68,7 +68,7 @@ public LazyPriorityQueueScheduler(final Iterator requests, final Handle
*
* @param requests An iterator to obtain requests
*/
- public LazyPriorityQueueScheduler(final Iterator requests) {
+ public LazyPriorityJobQueue(final Iterator requests) {
this(requests, null);
}
diff --git a/src/main/java/ai/preferred/venom/job/LazyScheduler.java b/src/main/java/ai/preferred/venom/job/LazyScheduler.java
index c0ca235..ba1c016 100644
--- a/src/main/java/ai/preferred/venom/job/LazyScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/LazyScheduler.java
@@ -19,22 +19,14 @@
import ai.preferred.venom.Handler;
import ai.preferred.venom.request.Request;
-import javax.annotation.Nonnull;
-import java.util.Collection;
import java.util.Iterator;
-import java.util.concurrent.TimeUnit;
/**
* Deprecated, will be removed in the next release.
- * Please use LazyPriorityQueueScheduler instead
+ * Please use LazyPriorityJobQueue instead.
*/
@Deprecated
-public class LazyScheduler implements QueueScheduler {
-
- /**
- * Link
- */
- private final LazyPriorityQueueScheduler lazyPriorityQueueScheduler;
+public class LazyScheduler extends LazyPriorityJobQueue {
/**
* Constructs an instance of lazy scheduler with a default handler.
@@ -43,7 +35,7 @@ public class LazyScheduler implements QueueScheduler {
* @param handler The default handler to use
*/
public LazyScheduler(final Iterator requests, final Handler handler) {
- lazyPriorityQueueScheduler = new LazyPriorityQueueScheduler(requests, handler);
+ super(requests, handler);
}
/**
@@ -52,148 +44,7 @@ public LazyScheduler(final Iterator requests, final Handler handler) {
* @param requests An iterator to obtain requests
*/
public LazyScheduler(final Iterator requests) {
- lazyPriorityQueueScheduler = new LazyPriorityQueueScheduler(requests);
- }
-
- @Override
- public final Scheduler getScheduler() {
- return lazyPriorityQueueScheduler.getScheduler();
- }
-
- @Override
- public final boolean add(@Nonnull final Job job) {
- return lazyPriorityQueueScheduler.add(job);
- }
-
- @Override
- public final boolean offer(@Nonnull final Job job) {
- return lazyPriorityQueueScheduler.offer(job);
- }
-
- @Override
- public final Job remove() {
- return lazyPriorityQueueScheduler.remove();
- }
-
- @Override
- public final Job poll() {
- return lazyPriorityQueueScheduler.poll();
- }
-
- @Override
- public final Job element() {
- return lazyPriorityQueueScheduler.element();
- }
-
- @Override
- public final Job peek() {
- return lazyPriorityQueueScheduler.peek();
- }
-
- @Override
- public final void put(@Nonnull final Job job) throws InterruptedException {
- lazyPriorityQueueScheduler.put(job);
- }
-
- @Override
- public final boolean offer(final Job job, final long timeout, @Nonnull final TimeUnit unit) throws InterruptedException {
- return lazyPriorityQueueScheduler.offer(job, timeout, unit);
- }
-
- @Override
- public final Job take() throws InterruptedException {
- return lazyPriorityQueueScheduler.take();
- }
-
- @Override
- public final Job poll(final long timeout, @Nonnull final TimeUnit unit) throws InterruptedException {
- return lazyPriorityQueueScheduler.poll(timeout, unit);
- }
-
- @Override
- public final int remainingCapacity() {
- return lazyPriorityQueueScheduler.remainingCapacity();
- }
-
- @Override
- public final boolean remove(final Object o) {
- return lazyPriorityQueueScheduler.remove(o);
+ super(requests);
}
- @Override
- public final boolean containsAll(@Nonnull final Collection c) {
- return lazyPriorityQueueScheduler.containsAll(c);
- }
-
- @Override
- public final boolean addAll(@Nonnull final Collection c) {
- return lazyPriorityQueueScheduler.addAll(c);
- }
-
- @Override
- public final boolean removeAll(@Nonnull final Collection c) {
- return lazyPriorityQueueScheduler.removeAll(c);
- }
-
- @Override
- public final boolean retainAll(@Nonnull final Collection c) {
- return lazyPriorityQueueScheduler.retainAll(c);
- }
-
- @Override
- public final void clear() {
- lazyPriorityQueueScheduler.clear();
- }
-
- @SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
- @Override
- public final boolean equals(final Object o) {
- return lazyPriorityQueueScheduler.equals(o);
- }
-
- @Override
- public final int hashCode() {
- return lazyPriorityQueueScheduler.hashCode();
- }
-
- @Override
- public final int size() {
- return lazyPriorityQueueScheduler.size();
- }
-
- @Override
- public final boolean isEmpty() {
- return lazyPriorityQueueScheduler.isEmpty();
- }
-
- @Override
- public final boolean contains(final Object o) {
- return lazyPriorityQueueScheduler.contains(o);
- }
-
- @Override
- public final Iterator iterator() {
- return lazyPriorityQueueScheduler.iterator();
- }
-
- @Override
- public final Object[] toArray() {
- return lazyPriorityQueueScheduler.toArray();
- }
-
- @SuppressWarnings("SuspiciousToArrayCall")
- @Override
- public final T[] toArray(@Nonnull final T[] a) {
- return lazyPriorityQueueScheduler.toArray(a);
- }
-
- @Override
- public final int drainTo(@Nonnull final Collection c) {
- return lazyPriorityQueueScheduler.drainTo(c);
- }
-
- @Override
- public final int drainTo(@Nonnull final Collection c, final int maxElements) {
- return lazyPriorityQueueScheduler.drainTo(c, maxElements);
- }
}
diff --git a/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java b/src/main/java/ai/preferred/venom/job/PriorityJobQueue.java
similarity index 93%
rename from src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
rename to src/main/java/ai/preferred/venom/job/PriorityJobQueue.java
index 5cfd048..9f67f11 100644
--- a/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/PriorityJobQueue.java
@@ -30,7 +30,7 @@
* @author Maksim Tkachenko
* @author Ween Jiann Lee
*/
-public class PriorityQueueScheduler extends AbstractPriorityQueueScheduler {
+public class PriorityJobQueue extends AbstractPriorityJobQueue {
@Override
public final Job poll(final long timeout, final @Nonnull TimeUnit unit) throws InterruptedException {
diff --git a/src/main/java/ai/preferred/venom/job/PriorityJobScheduler.java b/src/main/java/ai/preferred/venom/job/PriorityJobScheduler.java
new file mode 100644
index 0000000..d985000
--- /dev/null
+++ b/src/main/java/ai/preferred/venom/job/PriorityJobScheduler.java
@@ -0,0 +1,9 @@
+package ai.preferred.venom.job;
+
+/**
+ * For backwards compatibility.
+ */
+@Deprecated
+public class PriorityJobScheduler extends PriorityJobQueue {
+
+}
diff --git a/src/main/java/ai/preferred/venom/job/QueueScheduler.java b/src/main/java/ai/preferred/venom/job/QueueScheduler.java
deleted file mode 100644
index 20e9f63..0000000
--- a/src/main/java/ai/preferred/venom/job/QueueScheduler.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2018 Preferred.AI
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ai.preferred.venom.job;
-
-
-import java.util.concurrent.BlockingQueue;
-
-/**
- * This interface represents only the most basic of a scheduler.
- * It imposes no restrictions or particular details on the the
- * type of queue, and allows for different future types to be returned.
- *
- * @author Maksim Tkachenko
- * @author Ween Jiann Lee
- */
-public interface QueueScheduler extends BlockingQueue {
-
- /**
- * Get the scheduler to add jobs.
- *
- * @return an instance of Scheduler
- */
- Scheduler getScheduler();
-
-}
diff --git a/src/main/java/ai/preferred/venom/job/Scheduler.java b/src/main/java/ai/preferred/venom/job/Scheduler.java
index 4e323cf..e286d18 100644
--- a/src/main/java/ai/preferred/venom/job/Scheduler.java
+++ b/src/main/java/ai/preferred/venom/job/Scheduler.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2018 Preferred.AI
+ * Copyright (c) 2019 Preferred.AI
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,8 +18,11 @@
import ai.preferred.venom.Handler;
import ai.preferred.venom.request.Request;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.validation.constraints.NotNull;
+import java.util.concurrent.BlockingQueue;
/**
* This interface represents only the adding part a scheduler.
@@ -27,7 +30,26 @@
* @author Maksim Tkachenko
* @author Ween Jiann Lee
*/
-public interface Scheduler {
+public class Scheduler {
+
+ /**
+ * Logger.
+ */
+ private static final Logger LOGGER = LoggerFactory.getLogger(Scheduler.class);
+
+ /**
+ * The queue used for this scheduler.
+ */
+ private final BlockingQueue queue;
+
+ /**
+ * Constructs an instance of Scheduler.
+ *
+ * @param queue an instance of BlockingQueue
+ */
+ public Scheduler(final BlockingQueue queue) {
+ this.queue = queue;
+ }
/**
* Adds a request to the queue.
@@ -39,7 +61,12 @@ public interface Scheduler {
* @param handler handler to be used to parse the request.
* @param jobAttributes attributes to insert to the job.
*/
- void add(@NotNull Request request, Handler handler, @NotNull JobAttribute... jobAttributes);
+ public final void add(final @NotNull Request request, final Handler handler,
+ final @NotNull JobAttribute... jobAttributes) {
+ final Job job = new Job(request, handler, jobAttributes);
+ queue.add(job);
+ LOGGER.debug("Job {} - {} added to queue.", job.toString(), request.getUrl());
+ }
/**
* Adds a request to the queue.
@@ -50,7 +77,9 @@ public interface Scheduler {
* @param request request to fetch when dequeued.
* @param jobAttributes attributes to insert to the job.
*/
- void add(@NotNull Request request, @NotNull JobAttribute... jobAttributes);
+ public final void add(final @NotNull Request request, final @NotNull JobAttribute... jobAttributes) {
+ add(request, null, jobAttributes);
+ }
/**
* Adds a request to the queue.
@@ -61,7 +90,9 @@ public interface Scheduler {
* @param request request to fetch when dequeued.
* @param handler handler to be used to parse the request.
*/
- void add(@NotNull Request request, @NotNull Handler handler);
+ public final void add(final Request request, final @NotNull Handler handler) {
+ add(request, handler, new JobAttribute[0]);
+ }
/**
* Adds a request to the queue.
@@ -72,7 +103,9 @@ public interface Scheduler {
*
* @param request request to fetch when dequeued.
*/
- void add(@NotNull Request request);
+ public final void add(final @NotNull Request request) {
+ add(request, null, new JobAttribute[0]);
+ }
/**
* Adds a request to the queue. Will be removed in the next release.
@@ -86,8 +119,9 @@ public interface Scheduler {
* @param p initial priority of the request
* @param pf the minimum (floor) priority of this request
*/
- @Deprecated
- void add(@NotNull Request r, Handler h, Priority p, Priority pf);
+ public final void add(final @NotNull Request r, final Handler h, final Priority p, final Priority pf) {
+ add(r, h, new PriorityJobAttribute(p, pf));
+ }
/**
* Adds a request to the queue. Will be removed in the next release.
@@ -100,8 +134,9 @@ public interface Scheduler {
* @param h handler to be used to parse the request
* @param p initial priority of the request
*/
- @Deprecated
- void add(@NotNull Request r, Handler h, Priority p);
+ public final void add(final @NotNull Request r, final Handler h, final Priority p) {
+ add(r, h, p, Priority.FLOOR);
+ }
/**
* Adds a request to the queue. Will be removed in the next release.
@@ -116,7 +151,9 @@ public interface Scheduler {
* @param pf the minimum (floor) priority of this request
*/
@Deprecated
- void add(@NotNull Request r, Priority p, Priority pf);
+ public final void add(final @NotNull Request r, final Priority p, final Priority pf) {
+ add(r, null, p, pf);
+ }
/**
* Adds a request to the queue. Will be removed in the next release.
@@ -130,6 +167,7 @@ public interface Scheduler {
* @param p initial priority of the request
*/
@Deprecated
- void add(@NotNull Request r, Priority p);
-
+ public final void add(final @NotNull Request r, final Priority p) {
+ add(r, (Handler) null, p);
+ }
}
diff --git a/src/test/java/ai/preferred/venom/CrawlerBuilderTest.java b/src/test/java/ai/preferred/venom/CrawlerBuilderTest.java
index 7d6211f..27c4300 100644
--- a/src/test/java/ai/preferred/venom/CrawlerBuilderTest.java
+++ b/src/test/java/ai/preferred/venom/CrawlerBuilderTest.java
@@ -17,7 +17,7 @@
package ai.preferred.venom;
import ai.preferred.venom.fetcher.AsyncFetcher;
-import ai.preferred.venom.job.FIFOQueueScheduler;
+import ai.preferred.venom.job.FIFOJobQueue;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -71,8 +71,8 @@ void testSetPropRetainProxy() {
@Test
void testSetScheduler() {
- Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setScheduler(null));
- Crawler.builder().setScheduler(new FIFOQueueScheduler());
+ Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setJobQueue(null));
+ Crawler.builder().setJobQueue(new FIFOJobQueue());
}
@Test
diff --git a/src/test/java/ai/preferred/venom/CrawlerTest.java b/src/test/java/ai/preferred/venom/CrawlerTest.java
index a85f22f..0fd7bc5 100644
--- a/src/test/java/ai/preferred/venom/CrawlerTest.java
+++ b/src/test/java/ai/preferred/venom/CrawlerTest.java
@@ -18,8 +18,8 @@
import ai.preferred.venom.fetcher.FakeFetcher;
import ai.preferred.venom.fetcher.Fetcher;
-import ai.preferred.venom.job.FIFOQueueScheduler;
-import ai.preferred.venom.job.LazyPriorityQueueScheduler;
+import ai.preferred.venom.job.FIFOJobQueue;
+import ai.preferred.venom.job.LazyPriorityJobQueue;
import ai.preferred.venom.request.Request;
import ai.preferred.venom.request.VRequest;
import org.apache.http.HttpHost;
@@ -59,7 +59,7 @@ public void testCrawler() throws Exception {
.setFetcher(fetcher)
.setMaxConnections(1)
.setMaxTries(2)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSleepScheduler(new SleepScheduler(0))
.build()
.start()) {
@@ -85,7 +85,7 @@ public void testCrawlerStartAndClose() throws Exception {
.setFetcher(fetcher)
.setMaxConnections(1)
.setMaxTries(2)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSleepScheduler(new SleepScheduler(0))
.build();
@@ -113,7 +113,7 @@ public void testRetry() throws Exception {
.setFetcher(fetcher)
.setMaxConnections(1)
.setMaxTries(5)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSleepScheduler(new SleepScheduler(0))
.build()
.start()) {
@@ -141,7 +141,7 @@ public void testMaxTries() throws Exception {
.setFetcher(fetcher)
.setMaxConnections(1)
.setMaxTries(5)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSleepScheduler(new SleepScheduler(0))
.build()
.start()) {
@@ -174,7 +174,7 @@ public void testProxyProportionRemoved() throws Exception {
.setMaxConnections(1)
.setPropRetainProxy(0.2)
.setMaxTries(5)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSleepScheduler(new SleepScheduler(0))
.build()
.start()) {
@@ -207,7 +207,7 @@ public void testProxyProportionRetained() throws Exception {
.setMaxConnections(1)
.setPropRetainProxy(0.2)
.setMaxTries(5)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSleepScheduler(new SleepScheduler(0))
.build()
.start()) {
@@ -239,7 +239,7 @@ public void testLazySchedulerIntegration() throws Exception {
.setMaxConnections(1)
.setPropRetainProxy(0.2)
.setMaxTries(5)
- .setScheduler(new LazyPriorityQueueScheduler(requests.iterator(), handler))
+ .setJobQueue(new LazyPriorityJobQueue(requests.iterator(), handler))
.setSleepScheduler(new SleepScheduler(0))
.build()
.start()) {
@@ -261,7 +261,7 @@ public void testUrlRouterIntegration() throws Exception {
try (final Crawler crawler = Crawler.builder()
.setFetcher(fetcher)
.setMaxTries(1)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSleepScheduler(new SleepScheduler(0))
.setHandlerRouter(urlRouter)
.build()
@@ -285,7 +285,7 @@ public void testFatalHandlerException() {
.setFetcher(fetcher)
.setMaxTries(1)
.setMaxConnections(1)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSleepScheduler(new SleepScheduler(0))
.build()
.start()) {
@@ -324,7 +324,7 @@ public void testSessionIntegration() throws Exception {
try (final Crawler crawler = Crawler.builder()
.setFetcher(fetcher)
.setMaxTries(1)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSleepScheduler(new SleepScheduler(0))
.setSession(session)
.build()
@@ -355,7 +355,7 @@ public void testInterruptAndClose() throws Exception {
final Crawler crawler = Crawler.builder()
.setFetcher(fetcher)
.setMaxTries(1)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSession(session)
.build()
.start();
@@ -379,7 +379,7 @@ public void testStopCodeException() throws Exception {
.setFetcher(fetcher)
.setMaxConnections(1)
.setMaxTries(5)
- .setScheduler(new FIFOQueueScheduler())
+ .setJobQueue(new FIFOJobQueue())
.setSleepScheduler(new SleepScheduler(0))
.build()
.start()) {
diff --git a/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/FIFOJobQueueTest.java
similarity index 67%
rename from src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
rename to src/test/java/ai/preferred/venom/job/FIFOJobQueueTest.java
index 3edc417..fc3230f 100644
--- a/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/FIFOJobQueueTest.java
@@ -23,63 +23,63 @@
import java.util.concurrent.TimeUnit;
-class FIFOQueueSchedulerTest {
+class FIFOJobQueueTest {
private final String url = "https://venom.preferred.ai";
private final VRequest vRequest = new VRequest(url);
private final Job job = new Job(vRequest);
- private FIFOQueueScheduler scheduler;
+ private FIFOJobQueue jobQueue;
@BeforeEach
void initEach() {
- scheduler = new FIFOQueueScheduler();
+ jobQueue = new FIFOJobQueue();
}
@Test
void testAddRequest() {
- scheduler.add(job);
- final Job pollJob = scheduler.poll();
+ jobQueue.add(job);
+ final Job pollJob = jobQueue.poll();
Assertions.assertEquals(job, pollJob);
}
@Test
void testPutRequest() throws InterruptedException {
- scheduler.put(job);
- final Job pollJob = scheduler.poll();
+ jobQueue.put(job);
+ final Job pollJob = jobQueue.poll();
Assertions.assertEquals(job, pollJob);
}
@Test
void testOfferRequest() {
- scheduler.offer(job);
- final Job pollJob = scheduler.poll();
+ jobQueue.offer(job);
+ final Job pollJob = jobQueue.poll();
Assertions.assertEquals(job, pollJob);
}
@Test
void testOfferTimeoutRequest() throws InterruptedException {
- scheduler.offer(job, 1L, TimeUnit.NANOSECONDS);
- final Job pollJob = scheduler.poll();
+ jobQueue.offer(job, 1L, TimeUnit.NANOSECONDS);
+ final Job pollJob = jobQueue.poll();
Assertions.assertEquals(job, pollJob);
}
@Test
void testPollTimeout() throws InterruptedException {
- scheduler.add(job);
- final Job pollJob = scheduler.poll(1L, TimeUnit.NANOSECONDS);
+ jobQueue.add(job);
+ final Job pollJob = jobQueue.poll(1L, TimeUnit.NANOSECONDS);
Assertions.assertEquals(job, pollJob);
}
@Test
void testFIFOQueue() {
final Job job = new Job(vRequest, null, new PriorityJobAttribute(Priority.HIGH));
- scheduler.add(job);
- scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.HIGHEST)));
- scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.DEFAULT)));
- scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.LOW)));
+ jobQueue.add(job);
+ jobQueue.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.HIGHEST)));
+ jobQueue.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.DEFAULT)));
+ jobQueue.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.LOW)));
- final Job pollJob = scheduler.poll();
+ final Job pollJob = jobQueue.poll();
Assertions.assertEquals(job, pollJob);
}
diff --git a/src/test/java/ai/preferred/venom/job/LazyPriorityQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/LazyPriorityJobQueueTest.java
similarity index 82%
rename from src/test/java/ai/preferred/venom/job/LazyPriorityQueueSchedulerTest.java
rename to src/test/java/ai/preferred/venom/job/LazyPriorityJobQueueTest.java
index 843f4dc..e61315d 100644
--- a/src/test/java/ai/preferred/venom/job/LazyPriorityQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/LazyPriorityJobQueueTest.java
@@ -27,23 +27,23 @@
import java.util.List;
import java.util.concurrent.TimeUnit;
-class LazyPriorityQueueSchedulerTest {
+class LazyPriorityJobQueueTest {
private final String url = "https://venom.preferred.ai";
private final VRequest vRequest = new VRequest(url);
private final Job job = new Job(vRequest);
- private LazyPriorityQueueScheduler scheduler;
+ private LazyPriorityJobQueue jobQueue;
@BeforeEach
void initEach() {
- scheduler = new LazyPriorityQueueScheduler(null);
+ jobQueue = new LazyPriorityJobQueue(null);
}
@Test
void testAddRequest() {
- scheduler.add(job);
- final Job pollJob = scheduler.poll();
+ jobQueue.add(job);
+ final Job pollJob = jobQueue.poll();
Assertions.assertNotNull(pollJob);
Assertions.assertEquals(job, pollJob);
Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
@@ -51,8 +51,8 @@ void testAddRequest() {
@Test
void testPutRequest() throws InterruptedException {
- scheduler.put(job);
- final Job pollJob = scheduler.poll();
+ jobQueue.put(job);
+ final Job pollJob = jobQueue.poll();
Assertions.assertNotNull(pollJob);
Assertions.assertEquals(job, pollJob);
Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
@@ -60,8 +60,8 @@ void testPutRequest() throws InterruptedException {
@Test
void testOfferRequest() {
- scheduler.offer(job);
- final Job pollJob = scheduler.poll();
+ jobQueue.offer(job);
+ final Job pollJob = jobQueue.poll();
Assertions.assertNotNull(pollJob);
Assertions.assertEquals(job, pollJob);
Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
@@ -69,8 +69,8 @@ void testOfferRequest() {
@Test
void testOfferTimeoutRequest() throws InterruptedException {
- scheduler.offer(job, 1L, TimeUnit.NANOSECONDS);
- final Job pollJob = scheduler.poll();
+ jobQueue.offer(job, 1L, TimeUnit.NANOSECONDS);
+ final Job pollJob = jobQueue.poll();
Assertions.assertNotNull(pollJob);
Assertions.assertEquals(job, pollJob);
Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
@@ -78,8 +78,8 @@ void testOfferTimeoutRequest() throws InterruptedException {
@Test
void testPollTimeout() throws InterruptedException {
- scheduler.add(job);
- final Job pollJob = scheduler.poll(1L, TimeUnit.NANOSECONDS);
+ jobQueue.add(job);
+ final Job pollJob = jobQueue.poll(1L, TimeUnit.NANOSECONDS);
Assertions.assertEquals(job, pollJob);
}
@@ -97,7 +97,7 @@ void testIterator() {
};
- final LazyPriorityQueueScheduler scheduler = new LazyPriorityQueueScheduler(requests.iterator(), handler);
+ final LazyPriorityJobQueue scheduler = new LazyPriorityJobQueue(requests.iterator(), handler);
final Job pollJob = scheduler.poll();
Assertions.assertNotNull(pollJob);
@@ -120,7 +120,7 @@ void testLazyQueue() {
requests.add(vRequestNeg);
requests.add(vRequestNeg);
- final LazyPriorityQueueScheduler scheduler = new LazyPriorityQueueScheduler(requests.iterator());
+ final LazyPriorityJobQueue scheduler = new LazyPriorityJobQueue(requests.iterator());
final Job job = new Job(vRequest);
scheduler.add(job);
@@ -132,7 +132,7 @@ void testLazyQueue() {
void testIsEmpty() {
final List requests = new ArrayList<>();
requests.add(vRequest);
- final LazyPriorityQueueScheduler scheduler = new LazyPriorityQueueScheduler(requests.iterator());
+ final LazyPriorityJobQueue scheduler = new LazyPriorityJobQueue(requests.iterator());
scheduler.add(job);
Assertions.assertFalse(scheduler.isEmpty());
diff --git a/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/PriorityJobQueueTest.java
similarity index 72%
rename from src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
rename to src/test/java/ai/preferred/venom/job/PriorityJobQueueTest.java
index 369d161..4860e90 100644
--- a/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/PriorityJobQueueTest.java
@@ -23,23 +23,23 @@
import java.util.concurrent.TimeUnit;
-class PriorityQueueSchedulerTest {
+class PriorityJobQueueTest {
private final String url = "https://venom.preferred.ai";
private final VRequest vRequest = new VRequest(url);
private final Job job = new Job(vRequest);
- private PriorityQueueScheduler scheduler;
+ private PriorityJobQueue jobQueue;
@BeforeEach
void initEach() {
- scheduler = new PriorityQueueScheduler();
+ jobQueue = new PriorityJobQueue();
}
@Test
void testAddRequest() {
- scheduler.add(job);
- final Job pollJob = scheduler.poll();
+ jobQueue.add(job);
+ final Job pollJob = jobQueue.poll();
Assertions.assertNotNull(pollJob);
Assertions.assertEquals(job, pollJob);
Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
@@ -47,8 +47,8 @@ void testAddRequest() {
@Test
void testPutRequest() throws InterruptedException {
- scheduler.put(job);
- final Job pollJob = scheduler.poll();
+ jobQueue.put(job);
+ final Job pollJob = jobQueue.poll();
Assertions.assertNotNull(pollJob);
Assertions.assertEquals(job, pollJob);
Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
@@ -56,8 +56,8 @@ void testPutRequest() throws InterruptedException {
@Test
void testOfferRequest() {
- scheduler.offer(job);
- final Job pollJob = scheduler.poll();
+ jobQueue.offer(job);
+ final Job pollJob = jobQueue.poll();
Assertions.assertNotNull(pollJob);
Assertions.assertEquals(job, pollJob);
Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
@@ -65,8 +65,8 @@ void testOfferRequest() {
@Test
void testOfferTimeoutRequest() throws InterruptedException {
- scheduler.offer(job, 1L, TimeUnit.NANOSECONDS);
- final Job pollJob = scheduler.poll();
+ jobQueue.offer(job, 1L, TimeUnit.NANOSECONDS);
+ final Job pollJob = jobQueue.poll();
Assertions.assertNotNull(pollJob);
Assertions.assertEquals(job, pollJob);
Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
@@ -74,20 +74,20 @@ void testOfferTimeoutRequest() throws InterruptedException {
@Test
void testPollTimeout() throws InterruptedException {
- scheduler.add(job);
- final Job pollJob = scheduler.poll(1L, TimeUnit.NANOSECONDS);
+ jobQueue.add(job);
+ final Job pollJob = jobQueue.poll(1L, TimeUnit.NANOSECONDS);
Assertions.assertEquals(job, pollJob);
}
@Test
void testPriorityQueue() {
final Job job = new Job(vRequest, null, new PriorityJobAttribute(Priority.HIGHEST));
- scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.HIGH)));
- scheduler.add(job);
- scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.DEFAULT)));
- scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.LOW)));
+ jobQueue.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.HIGH)));
+ jobQueue.add(job);
+ jobQueue.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.DEFAULT)));
+ jobQueue.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.LOW)));
- final Job pollJob = scheduler.poll();
+ final Job pollJob = jobQueue.poll();
Assertions.assertEquals(pollJob, job);
}
diff --git a/src/test/java/ai/preferred/venom/job/JobSchedulerTest.java b/src/test/java/ai/preferred/venom/job/SchedulerTest.java
similarity index 72%
rename from src/test/java/ai/preferred/venom/job/JobSchedulerTest.java
rename to src/test/java/ai/preferred/venom/job/SchedulerTest.java
index 7b40190..659fdeb 100644
--- a/src/test/java/ai/preferred/venom/job/JobSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/SchedulerTest.java
@@ -21,7 +21,7 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-class JobSchedulerTest {
+class SchedulerTest {
private final String url = "https://venom.preferred.ai";
private final VRequest vRequest = new VRequest(url);
@@ -31,9 +31,10 @@ class JobSchedulerTest {
@Test
void testAddRequest() {
- final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
- scheduler.getScheduler().add(vRequest);
- final Job job = scheduler.poll();
+ final FIFOJobQueue jobQueue = new FIFOJobQueue();
+ final Scheduler scheduler = new Scheduler(jobQueue);
+ scheduler.add(vRequest);
+ final Job job = jobQueue.poll();
Assertions.assertNotNull(job);
Assertions.assertEquals(vRequest, job.getRequest());
Assertions.assertNull(job.getHandler());
@@ -41,9 +42,10 @@ void testAddRequest() {
@Test
void testAddRequestHandler() {
- final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
- scheduler.getScheduler().add(vRequest, handler);
- final Job job = scheduler.poll();
+ final FIFOJobQueue jobQueue = new FIFOJobQueue();
+ final Scheduler scheduler = new Scheduler(jobQueue);
+ scheduler.add(vRequest, handler);
+ final Job job = jobQueue.poll();
Assertions.assertNotNull(job);
Assertions.assertEquals(vRequest, job.getRequest());
Assertions.assertEquals(handler, job.getHandler());
@@ -51,10 +53,11 @@ void testAddRequestHandler() {
@Test
void testAddRequestJobAttribute() {
- final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
+ final FIFOJobQueue jobQueue = new FIFOJobQueue();
+ final Scheduler scheduler = new Scheduler(jobQueue);
final PriorityJobAttribute priorityJobAttribute = new PriorityJobAttribute();
- scheduler.getScheduler().add(vRequest, priorityJobAttribute);
- final Job job = scheduler.poll();
+ scheduler.add(vRequest, priorityJobAttribute);
+ final Job job = jobQueue.poll();
Assertions.assertNotNull(job);
Assertions.assertEquals(vRequest, job.getRequest());
Assertions.assertNull(job.getHandler());
@@ -63,10 +66,11 @@ void testAddRequestJobAttribute() {
@Test
void testAddRequestHandlerJobAttribute() {
- final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
+ final FIFOJobQueue jobQueue = new FIFOJobQueue();
+ final Scheduler scheduler = new Scheduler(jobQueue);
final PriorityJobAttribute priorityJobAttribute = new PriorityJobAttribute();
- scheduler.getScheduler().add(vRequest, handler, priorityJobAttribute);
- final Job job = scheduler.poll();
+ scheduler.add(vRequest, handler, priorityJobAttribute);
+ final Job job = jobQueue.poll();
Assertions.assertNotNull(job);
Assertions.assertEquals(vRequest, job.getRequest());
Assertions.assertEquals(handler, job.getHandler());