From 238ce6d068c60fa36a4c5276b2d2e3f27276aae0 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Fri, 19 Jul 2013 23:09:43 +0800 Subject: [PATCH 1/8] Fixed test --- .../metamorphosis/metaslave/SlaveConsumerZooKeeperUnitTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metamorphosis-server-wrapper/src/test/java/com/taobao/metamorphosis/metaslave/SlaveConsumerZooKeeperUnitTest.java b/metamorphosis-server-wrapper/src/test/java/com/taobao/metamorphosis/metaslave/SlaveConsumerZooKeeperUnitTest.java index 0569a906..aace0de2 100644 --- a/metamorphosis-server-wrapper/src/test/java/com/taobao/metamorphosis/metaslave/SlaveConsumerZooKeeperUnitTest.java +++ b/metamorphosis-server-wrapper/src/test/java/com/taobao/metamorphosis/metaslave/SlaveConsumerZooKeeperUnitTest.java @@ -296,7 +296,7 @@ private void checkTopic2( private void mockConnect(final String url) throws NotifyRemotingException, InterruptedException { this.remotingClient.connectWithRef(EasyMock.eq(url), EasyMock.anyObject()); EasyMock.expectLastCall(); - this.remotingClient.awaitReadyInterrupt(url, 10000); + this.remotingClient.awaitReadyInterrupt(url, 4000); EasyMock.expectLastCall(); } From d640908cfe64d072f46f0a5a852472b6ab339e78 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Fri, 19 Jul 2013 23:21:31 +0800 Subject: [PATCH 2/8] [maven-release-plugin] prepare release metamorphosis-all-1.4.6.2 --- metamorphosis-client-extension/pom.xml | 2 +- metamorphosis-client/pom.xml | 2 +- metamorphosis-commons/pom.xml | 2 +- metamorphosis-dashboard/pom.xml | 4 ++-- metamorphosis-server-wrapper/pom.xml | 2 +- metamorphosis-server/pom.xml | 2 +- metamorphosis-storm-spout/pom.xml | 2 +- metamorphosis-tools/pom.xml | 2 +- pom.xml | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/metamorphosis-client-extension/pom.xml b/metamorphosis-client-extension/pom.xml index f2e52eac..00ad59cd 100644 --- a/metamorphosis-client-extension/pom.xml +++ b/metamorphosis-client-extension/pom.xml @@ -3,7 +3,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2-SNAPSHOT + 1.4.6.2 4.0.0 diff --git a/metamorphosis-client/pom.xml b/metamorphosis-client/pom.xml index 5105252d..e475707a 100644 --- a/metamorphosis-client/pom.xml +++ b/metamorphosis-client/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2-SNAPSHOT + 1.4.6.2 4.0.0 metamorphosis-client diff --git a/metamorphosis-commons/pom.xml b/metamorphosis-commons/pom.xml index f08ffd51..b8e1496b 100644 --- a/metamorphosis-commons/pom.xml +++ b/metamorphosis-commons/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2-SNAPSHOT + 1.4.6.2 4.0.0 metamorphosis-commons diff --git a/metamorphosis-dashboard/pom.xml b/metamorphosis-dashboard/pom.xml index bcc9ae6b..7eab369c 100644 --- a/metamorphosis-dashboard/pom.xml +++ b/metamorphosis-dashboard/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2-SNAPSHOT + 1.4.6.2 4.0.0 metamorphosis-dashboard @@ -149,7 +149,7 @@ com.taobao.metamorphosis metamorphosis-server - 1.4.6.2-SNAPSHOT + 1.4.6.2 ring-mock diff --git a/metamorphosis-server-wrapper/pom.xml b/metamorphosis-server-wrapper/pom.xml index 9df25225..cc30d3b6 100644 --- a/metamorphosis-server-wrapper/pom.xml +++ b/metamorphosis-server-wrapper/pom.xml @@ -3,7 +3,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2-SNAPSHOT + 1.4.6.2 4.0.0 diff --git a/metamorphosis-server/pom.xml b/metamorphosis-server/pom.xml index 0ca791e6..0cb2c2ae 100644 --- a/metamorphosis-server/pom.xml +++ b/metamorphosis-server/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2-SNAPSHOT + 1.4.6.2 4.0.0 metamorphosis-server diff --git a/metamorphosis-storm-spout/pom.xml b/metamorphosis-storm-spout/pom.xml index 32ff824b..2539577e 100644 --- a/metamorphosis-storm-spout/pom.xml +++ b/metamorphosis-storm-spout/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2-SNAPSHOT + 1.4.6.2 4.0.0 metamorphosis-storm-spout diff --git a/metamorphosis-tools/pom.xml b/metamorphosis-tools/pom.xml index 83a97e57..10c7e035 100644 --- a/metamorphosis-tools/pom.xml +++ b/metamorphosis-tools/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2-SNAPSHOT + 1.4.6.2 4.0.0 metamorphosis-tools v${project.version} diff --git a/pom.xml b/pom.xml index c9bb3bd1..a6ff1557 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.taobao.metamorphosis metamorphosis-all pom - 1.4.6.2-SNAPSHOT + 1.4.6.2 taobao metamorphosis v${project.version} https://github.com/killme2008/Metamorphosis From 508f501316a5a4c820293a722cb8b6c23cddb060 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Fri, 19 Jul 2013 23:21:52 +0800 Subject: [PATCH 3/8] [maven-release-plugin] prepare for next development iteration --- metamorphosis-client-extension/pom.xml | 2 +- metamorphosis-client/pom.xml | 2 +- metamorphosis-commons/pom.xml | 2 +- metamorphosis-dashboard/pom.xml | 4 ++-- metamorphosis-server-wrapper/pom.xml | 2 +- metamorphosis-server/pom.xml | 2 +- metamorphosis-storm-spout/pom.xml | 2 +- metamorphosis-tools/pom.xml | 2 +- pom.xml | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/metamorphosis-client-extension/pom.xml b/metamorphosis-client-extension/pom.xml index 00ad59cd..c69b9399 100644 --- a/metamorphosis-client-extension/pom.xml +++ b/metamorphosis-client-extension/pom.xml @@ -3,7 +3,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2 + 1.4.6.3-SNAPSHOT 4.0.0 diff --git a/metamorphosis-client/pom.xml b/metamorphosis-client/pom.xml index e475707a..d23c9bfc 100644 --- a/metamorphosis-client/pom.xml +++ b/metamorphosis-client/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2 + 1.4.6.3-SNAPSHOT 4.0.0 metamorphosis-client diff --git a/metamorphosis-commons/pom.xml b/metamorphosis-commons/pom.xml index b8e1496b..25bccf12 100644 --- a/metamorphosis-commons/pom.xml +++ b/metamorphosis-commons/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2 + 1.4.6.3-SNAPSHOT 4.0.0 metamorphosis-commons diff --git a/metamorphosis-dashboard/pom.xml b/metamorphosis-dashboard/pom.xml index 7eab369c..af10b968 100644 --- a/metamorphosis-dashboard/pom.xml +++ b/metamorphosis-dashboard/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2 + 1.4.6.3-SNAPSHOT 4.0.0 metamorphosis-dashboard @@ -149,7 +149,7 @@ com.taobao.metamorphosis metamorphosis-server - 1.4.6.2 + 1.4.6.3-SNAPSHOT ring-mock diff --git a/metamorphosis-server-wrapper/pom.xml b/metamorphosis-server-wrapper/pom.xml index cc30d3b6..48ffe715 100644 --- a/metamorphosis-server-wrapper/pom.xml +++ b/metamorphosis-server-wrapper/pom.xml @@ -3,7 +3,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2 + 1.4.6.3-SNAPSHOT 4.0.0 diff --git a/metamorphosis-server/pom.xml b/metamorphosis-server/pom.xml index 0cb2c2ae..420f7230 100644 --- a/metamorphosis-server/pom.xml +++ b/metamorphosis-server/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2 + 1.4.6.3-SNAPSHOT 4.0.0 metamorphosis-server diff --git a/metamorphosis-storm-spout/pom.xml b/metamorphosis-storm-spout/pom.xml index 2539577e..0d78ec18 100644 --- a/metamorphosis-storm-spout/pom.xml +++ b/metamorphosis-storm-spout/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2 + 1.4.6.3-SNAPSHOT 4.0.0 metamorphosis-storm-spout diff --git a/metamorphosis-tools/pom.xml b/metamorphosis-tools/pom.xml index 10c7e035..92701856 100644 --- a/metamorphosis-tools/pom.xml +++ b/metamorphosis-tools/pom.xml @@ -2,7 +2,7 @@ metamorphosis-all com.taobao.metamorphosis - 1.4.6.2 + 1.4.6.3-SNAPSHOT 4.0.0 metamorphosis-tools v${project.version} diff --git a/pom.xml b/pom.xml index a6ff1557..66585a00 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.taobao.metamorphosis metamorphosis-all pom - 1.4.6.2 + 1.4.6.3-SNAPSHOT taobao metamorphosis v${project.version} https://github.com/killme2008/Metamorphosis From 2b2a5a09530c01154ceb9ac81fb9d87d8a823f34 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Sat, 20 Jul 2013 00:19:54 +0800 Subject: [PATCH 4/8] Release 1.4.6.2 --- README.md | 1 + metamorphosis-example/pom.xml | 6 +++--- metamorphosis-storm-spout/README.md | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 42290ee9..4a9da787 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ ![Logo](http://photo.yupoo.com/killme2008/CLRQoBA9/medish.jpg) #新闻 +* MetaQ 1.4.6.2发布。[更新日志](https://github.com/killme2008/Metamorphosis/wiki/ReleaseNotes) * MetaQ 1.4.6.1发布。[更新日志](https://github.com/killme2008/Metamorphosis/wiki/ReleaseNotes) * MetaQ 1.4.5.1发布。[更新日志](https://github.com/killme2008/Metamorphosis/wiki/ReleaseNotes) * MetaQ 1.4.5发布。[更新日志](https://github.com/killme2008/Metamorphosis/wiki/ReleaseNotes) diff --git a/metamorphosis-example/pom.xml b/metamorphosis-example/pom.xml index 6426fd94..b078c4cd 100644 --- a/metamorphosis-example/pom.xml +++ b/metamorphosis-example/pom.xml @@ -37,17 +37,17 @@ com.taobao.metamorphosis metamorphosis-client - 1.4.6.1 + 1.4.6.2 com.taobao.metamorphosis metamorphosis-storm-spout - 1.4.6.1 + 1.4.6.2 com.taobao.metamorphosis metamorphosis-client-extension - 1.4.6.1 + 1.4.6.2 org.springframework diff --git a/metamorphosis-storm-spout/README.md b/metamorphosis-storm-spout/README.md index a3cb77a6..fe84054b 100644 --- a/metamorphosis-storm-spout/README.md +++ b/metamorphosis-storm-spout/README.md @@ -6,6 +6,6 @@ metamorphosis作为twitter strom spout数据源的支持项目 com.taobao.metamorphosis storm-metamorphosis-spout - 1.4.6.1 + 1.4.6.2 From cfbba55954c29a1ce70f5b8bcf594747fff1a3d9 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Sat, 20 Jul 2013 18:45:37 +0800 Subject: [PATCH 5/8] Use cpus+2 to load message stores --- .../metamorphosis/server/store/MessageStoreManager.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/metamorphosis-server/src/main/java/com/taobao/metamorphosis/server/store/MessageStoreManager.java b/metamorphosis-server/src/main/java/com/taobao/metamorphosis/server/store/MessageStoreManager.java index 4aec643c..0849aa75 100644 --- a/metamorphosis-server/src/main/java/com/taobao/metamorphosis/server/store/MessageStoreManager.java +++ b/metamorphosis-server/src/main/java/com/taobao/metamorphosis/server/store/MessageStoreManager.java @@ -317,7 +317,7 @@ private void loadDataDir(final MetaConfig metaConfig, final File dir) throws IOE log.warn("Begin to scan data path:" + dir.getAbsolutePath()); final long start = System.currentTimeMillis(); final File[] ls = dir.listFiles(); - int nThreads = Runtime.getRuntime().availableProcessors() + 1; + int nThreads = Runtime.getRuntime().availableProcessors() + 2; ExecutorService executor = Executors.newFixedThreadPool(nThreads); int count = 0; List> tasks = new ArrayList>(); @@ -354,6 +354,7 @@ public MessageStore call() throws Exception { else { this.loadStores(tasks); } + tasks.clear(); } } } @@ -384,7 +385,6 @@ private void loadStores(List> tasks) throws IOException, throw new IllegalStateException(e); } } - tasks.clear(); } @@ -409,7 +409,6 @@ private void loadStoresInParallel(ExecutorService executor, List Date: Wed, 24 Jul 2013 23:41:36 +0800 Subject: [PATCH 6/8] Register shutdown hook at setup --- .../server/assembly/MetaMorphosisBroker.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/metamorphosis-server/src/main/java/com/taobao/metamorphosis/server/assembly/MetaMorphosisBroker.java b/metamorphosis-server/src/main/java/com/taobao/metamorphosis/server/assembly/MetaMorphosisBroker.java index f0c49cd8..01ca872e 100644 --- a/metamorphosis-server/src/main/java/com/taobao/metamorphosis/server/assembly/MetaMorphosisBroker.java +++ b/metamorphosis-server/src/main/java/com/taobao/metamorphosis/server/assembly/MetaMorphosisBroker.java @@ -148,6 +148,8 @@ public void setBrokerProcessor(final CommandProcessor brokerProcessor) { public MetaMorphosisBroker(final MetaConfig metaConfig) { super(); + this.shutdownHook = new ShutdownHook(); + Runtime.getRuntime().addShutdownHook(this.shutdownHook); this.metaConfig = metaConfig; this.remotingServer = newRemotingServer(metaConfig); this.executorsManager = new ExecutorsManager(metaConfig); @@ -174,8 +176,6 @@ public MetaMorphosisBroker(final MetaConfig metaConfig) { this.brokerProcessor = new TransactionalCommandProcessor(metaConfig, this.storeManager, this.idWorker, next, transactionStore, this.statsManager); - this.shutdownHook = new ShutdownHook(); - Runtime.getRuntime().addShutdownHook(this.shutdownHook); MetaMBeanServer.registMBean(this, null); } @@ -301,7 +301,12 @@ public synchronized void stop() { } if (!this.runShutdownHook && this.shutdownHook != null) { - Runtime.getRuntime().removeShutdownHook(this.shutdownHook); + try { + Runtime.getRuntime().removeShutdownHook(this.shutdownHook); + } + catch (Exception e) { + // ignore + } } this.brokerProcessor.dispose(); From cbfc53477dc93ef15fcbeb6fd23a3b9e26185c69 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Thu, 25 Jul 2013 00:57:55 +0800 Subject: [PATCH 7/8] Don't close connections,TODO: concurrent issues --- .../metamorphosis/client/consumer/ConsumerZooKeeper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/consumer/ConsumerZooKeeper.java b/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/consumer/ConsumerZooKeeper.java index 82d8b47e..a1cd87a8 100644 --- a/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/consumer/ConsumerZooKeeper.java +++ b/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/consumer/ConsumerZooKeeper.java @@ -702,6 +702,7 @@ boolean rebalance() throws InterruptedException, Exception { this.fetchManager.stopFetchRunner(); // closed all connections to old brokers. this.closeOldBrokersConnections(); + this.commitOffsets(); this.updateFetchRunner(cluster); this.oldCluster = cluster; } @@ -789,7 +790,8 @@ boolean rebalance() throws InterruptedException, Exception { private void closeOldBrokersConnections() throws NotifyRemotingException { for (Broker old : this.oldBrokerSet) { - ConsumerZooKeeper.this.remotingClient.closeWithRef(old.getZKString(), this, false); + // ConsumerZooKeeper.this.remotingClient.closeWithRef(old.getZKString(), + // this, false); log.warn("Closed " + old.getZKString()); } } From 478beab4eff05bfe0453da7c809635a42b24047c Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Thu, 25 Jul 2013 01:34:23 +0800 Subject: [PATCH 8/8] Upgrade gecko to 1.1.5-SNAPSHOT and await connections in group to be closed --- .../client/RemotingClientWrapper.java | 27 ++++++++++++++++++- .../client/consumer/ConsumerZooKeeper.java | 3 +-- pom.xml | 2 +- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/RemotingClientWrapper.java b/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/RemotingClientWrapper.java index 2f4d0cb5..ef206729 100644 --- a/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/RemotingClientWrapper.java +++ b/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/RemotingClientWrapper.java @@ -129,13 +129,38 @@ public void closeWithRef(final String url, Object ref, final boolean allowReconn synchronized (refs) { refs.remove(ref); if (refs.isEmpty() || this.isOnlyMe(refs)) { - this.remotingClient.close(url, allowReconnect); + int times = 0; + while (times++ < 3) { + try { + this.remotingClient.close(url, allowReconnect); + this.remotingClient.awaitClosed(url, 5000); + break; + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + catch (TimeoutException e) { + // ignore + } + } } } } } + @Override + public void awaitClosed(String arg0, long arg1) throws InterruptedException, TimeoutException { + this.remotingClient.awaitClosed(arg0, arg1); + } + + + @Override + public void awaitClosed(String arg0) throws InterruptedException, TimeoutException { + this.remotingClient.awaitClosed(arg0); + } + + private boolean isOnlyMe(final Set refs) { return refs.size() == 1 && refs.contains(this); } diff --git a/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/consumer/ConsumerZooKeeper.java b/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/consumer/ConsumerZooKeeper.java index a1cd87a8..21b2e714 100644 --- a/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/consumer/ConsumerZooKeeper.java +++ b/metamorphosis-client/src/main/java/com/taobao/metamorphosis/client/consumer/ConsumerZooKeeper.java @@ -790,8 +790,7 @@ boolean rebalance() throws InterruptedException, Exception { private void closeOldBrokersConnections() throws NotifyRemotingException { for (Broker old : this.oldBrokerSet) { - // ConsumerZooKeeper.this.remotingClient.closeWithRef(old.getZKString(), - // this, false); + ConsumerZooKeeper.this.remotingClient.closeWithRef(old.getZKString(), this, false); log.warn("Closed " + old.getZKString()); } } diff --git a/pom.xml b/pom.xml index 66585a00..b62beb20 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.taobao.gecko gecko - 1.1.4 + 1.1.5-SNAPSHOT com.taobao.notify