Hazelcast instance used for 2nd Hibernate cache cannot connect stably in Kubernetes
.everyoneloves__top-leaderboard:empty,.everyoneloves__mid-leaderboard:empty,.everyoneloves__bot-mid-leaderboard:empty{ height:90px;width:728px;box-sizing:border-box;
}
Iam using embedded Hazelcast deployed in Kubernetes(GKE) to enable 2nd level cache Hibernate in Spring Boot application. Spingboot app connect to Spanner by using spanner-jdbc (https://github.com/olavloite/spanner-jdbc).
I deploy 1 service (replicas=2) to k8s, however hazelcast instances located in 2 pods cannot connect stably (it works when server starts but then seem disconnected after, please the below log). This causes 2nd level cache cannot be shared between 2 pods. Anyone can help this situation? Thank you.
<groupId>com.hazelcast.poc</groupId>
<artifactId>hazelcast-poc</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>hazelcast-poc</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>nl.topicus</groupId>
<artifactId>spanner-jdbc</artifactId>
<version>1.0.8</version>
</dependency>
<dependency>
<groupId>nl.topicus</groupId>
<artifactId>spanner-hibernate</artifactId>
<version>0.7</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-rest</artifactId>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast</artifactId>
<version>3.11</version>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast-hibernate5</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast-kubernetes</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.8.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.uuid</groupId>
<artifactId>java-uuid-generator</artifactId>
<version>3.1.5</version>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-java8</artifactId>
<version>${hibernate.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.0.1.RELEASE</version>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
</plugins>
</build>
Springboot Application.java:
public static void main(String args) throws IOException {
CloudSpannerDriver.setLogLevel(CloudSpannerDriver.DEBUG);
Writer writer = new FileWriter("jdbc.log", false);
DriverManager.setLogWriter(new PrintWriter(writer));SpringApplication.run(Application.class, args);
}
@Bean
public Config hazelCastConfig(){
Config config = new Config();
config
.addMapConfig(
new MapConfig()
.setName("employees")
.setMaxSizeConfig(new MaxSizeConfig(200, MaxSizeConfig.MaxSizePolicy.FREE_HEAP_SIZE))
.setEvictionPolicy(EvictionPolicy.LRU)
.setTimeToLiveSeconds(20000));
JoinConfig joinConfig = config.getNetworkConfig().getJoin();
joinConfig.getMulticastConfig().setEnabled(false);
joinConfig.getKubernetesConfig().setEnabled(true).setProperty("namespace", "default");
return config;
}
}
deployment.yaml:
apiVersion: apps/v1
kind: Deployment
metadata:
name: hazelcast-embedded-0
spec:
replicas: 2
selector:
matchLabels:
app: hazelcast-embedded-0
template:
metadata:
labels:
app: hazelcast-embedded-0
spec:
volumes:
- name: hazelcast-key
secret:
secretName: hazelcast-key
containers:
- name: hazelcast-embedded
image: gcr.io/testproject-221915/hazelcast- poc@sha256:5a103d7e2e56c46e9e42617232e472891bda5d121139fe7e41057596bd71bf00
imagePullPolicy: Always
volumeMounts:
- name: hazelcast-key
mountPath: /var/secrets/google
env:
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /var/secrets/google/key.json
ports:
- containerPort: 5701
- containerPort: 8080
apiVersion: v1
kind: Service
metadata:
name: hazelcast-embedded-0
spec:
type: LoadBalancer
selector:
app: hazelcast-embedded-0
ports:
- name: hazelcast
port: 5701
- name: app
port: 8080
Spanner Datasource file:
@Bean
public LocalContainerEntityManagerFactoryBean spannerEntityManager() {
LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean();
em.setDataSource(spannerDataSource());
em.setPersistenceUnitName("SpannerPU");
HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter();
em.setJpaVendorAdapter(vendorAdapter);
HashMap<String, Object> properties = new HashMap<>();
properties.put("hibernate.dialect", "nl.topicus.hibernate.dialect.CloudSpannerDialect");
properties.put("hibernate.temp.use_jdbc_metadata_defaults", false);
properties.put("hibernate.show_sql", showSql);
properties.put("hibernate.format_sql", formatSql);
properties.put("hibernate.cache.use_query_cache", true);
properties.put("hibernate.cache.use_second_level_cache", true);
properties.put("hibernate.cache.region.factory_class", "com.hazelcast.hibernate.HazelcastCacheRegionFactory");
if("development".equals(profileActive)) {
properties.put("hibernate.generate_statistics", statistic);
properties.put("org.hibernate.stat", hibernateStat);
}
em.setJpaPropertyMap(properties);
return em;
}
Application.properties
spanner.datasource.className=nl.topicus.jdbc.CloudSpannerDriver
spanner.datasource.url=jdbc:cloudspanner://;Project=testproject-
221915;Instance=test-hazelcast;Database=hazelcast
spanner.hibernate.dialect=
nl.topicus.hibernate.dialect.CloudSpannerDialect
spanner.hibernate.show-sql=true
spanner.hibernate.format_sql=true
spanner.datasource.hikari.maximum-pool-size=2 #depend on
microservice
spanner.datasource.hikari.minimum-idle=2
spanner.datasource.hikari.connection-timeout=60000
spring.profiles.active=dev
hibernate.generate_statistics=true
org.hibernate.stat=DEBUG
Log_pod1
Log_pod2
As you see, the second log has only 1 member whereas the first has 2 members.
I tested by issuing rest request to service which then distributed to each pods, the test result is sometimes cache are shared between two pods, sometimes not.
I expected 2nd Hibernate cache should be shared between 2 pods in the service
hibernate spring-boot kubernetes hazelcast
add a comment |
Iam using embedded Hazelcast deployed in Kubernetes(GKE) to enable 2nd level cache Hibernate in Spring Boot application. Spingboot app connect to Spanner by using spanner-jdbc (https://github.com/olavloite/spanner-jdbc).
I deploy 1 service (replicas=2) to k8s, however hazelcast instances located in 2 pods cannot connect stably (it works when server starts but then seem disconnected after, please the below log). This causes 2nd level cache cannot be shared between 2 pods. Anyone can help this situation? Thank you.
<groupId>com.hazelcast.poc</groupId>
<artifactId>hazelcast-poc</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>hazelcast-poc</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>nl.topicus</groupId>
<artifactId>spanner-jdbc</artifactId>
<version>1.0.8</version>
</dependency>
<dependency>
<groupId>nl.topicus</groupId>
<artifactId>spanner-hibernate</artifactId>
<version>0.7</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-rest</artifactId>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast</artifactId>
<version>3.11</version>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast-hibernate5</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast-kubernetes</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.8.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.uuid</groupId>
<artifactId>java-uuid-generator</artifactId>
<version>3.1.5</version>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-java8</artifactId>
<version>${hibernate.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.0.1.RELEASE</version>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
</plugins>
</build>
Springboot Application.java:
public static void main(String args) throws IOException {
CloudSpannerDriver.setLogLevel(CloudSpannerDriver.DEBUG);
Writer writer = new FileWriter("jdbc.log", false);
DriverManager.setLogWriter(new PrintWriter(writer));SpringApplication.run(Application.class, args);
}
@Bean
public Config hazelCastConfig(){
Config config = new Config();
config
.addMapConfig(
new MapConfig()
.setName("employees")
.setMaxSizeConfig(new MaxSizeConfig(200, MaxSizeConfig.MaxSizePolicy.FREE_HEAP_SIZE))
.setEvictionPolicy(EvictionPolicy.LRU)
.setTimeToLiveSeconds(20000));
JoinConfig joinConfig = config.getNetworkConfig().getJoin();
joinConfig.getMulticastConfig().setEnabled(false);
joinConfig.getKubernetesConfig().setEnabled(true).setProperty("namespace", "default");
return config;
}
}
deployment.yaml:
apiVersion: apps/v1
kind: Deployment
metadata:
name: hazelcast-embedded-0
spec:
replicas: 2
selector:
matchLabels:
app: hazelcast-embedded-0
template:
metadata:
labels:
app: hazelcast-embedded-0
spec:
volumes:
- name: hazelcast-key
secret:
secretName: hazelcast-key
containers:
- name: hazelcast-embedded
image: gcr.io/testproject-221915/hazelcast- poc@sha256:5a103d7e2e56c46e9e42617232e472891bda5d121139fe7e41057596bd71bf00
imagePullPolicy: Always
volumeMounts:
- name: hazelcast-key
mountPath: /var/secrets/google
env:
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /var/secrets/google/key.json
ports:
- containerPort: 5701
- containerPort: 8080
apiVersion: v1
kind: Service
metadata:
name: hazelcast-embedded-0
spec:
type: LoadBalancer
selector:
app: hazelcast-embedded-0
ports:
- name: hazelcast
port: 5701
- name: app
port: 8080
Spanner Datasource file:
@Bean
public LocalContainerEntityManagerFactoryBean spannerEntityManager() {
LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean();
em.setDataSource(spannerDataSource());
em.setPersistenceUnitName("SpannerPU");
HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter();
em.setJpaVendorAdapter(vendorAdapter);
HashMap<String, Object> properties = new HashMap<>();
properties.put("hibernate.dialect", "nl.topicus.hibernate.dialect.CloudSpannerDialect");
properties.put("hibernate.temp.use_jdbc_metadata_defaults", false);
properties.put("hibernate.show_sql", showSql);
properties.put("hibernate.format_sql", formatSql);
properties.put("hibernate.cache.use_query_cache", true);
properties.put("hibernate.cache.use_second_level_cache", true);
properties.put("hibernate.cache.region.factory_class", "com.hazelcast.hibernate.HazelcastCacheRegionFactory");
if("development".equals(profileActive)) {
properties.put("hibernate.generate_statistics", statistic);
properties.put("org.hibernate.stat", hibernateStat);
}
em.setJpaPropertyMap(properties);
return em;
}
Application.properties
spanner.datasource.className=nl.topicus.jdbc.CloudSpannerDriver
spanner.datasource.url=jdbc:cloudspanner://;Project=testproject-
221915;Instance=test-hazelcast;Database=hazelcast
spanner.hibernate.dialect=
nl.topicus.hibernate.dialect.CloudSpannerDialect
spanner.hibernate.show-sql=true
spanner.hibernate.format_sql=true
spanner.datasource.hikari.maximum-pool-size=2 #depend on
microservice
spanner.datasource.hikari.minimum-idle=2
spanner.datasource.hikari.connection-timeout=60000
spring.profiles.active=dev
hibernate.generate_statistics=true
org.hibernate.stat=DEBUG
Log_pod1
Log_pod2
As you see, the second log has only 1 member whereas the first has 2 members.
I tested by issuing rest request to service which then distributed to each pods, the test result is sometimes cache are shared between two pods, sometimes not.
I expected 2nd Hibernate cache should be shared between 2 pods in the service
hibernate spring-boot kubernetes hazelcast
Can you share the complete log (without| grep Member
)?
– Rafał Leszko
Jan 4 at 15:53
Hi @Rafał Leszko, I just updated the post with log of pod1 and pod2. I am trying to translate a line in log "Config seed port is 5701 and cluster size is 1. Some of the ports seem occupied!". Do you have any idea ?
– Tom Ho
Jan 4 at 16:35
add a comment |
Iam using embedded Hazelcast deployed in Kubernetes(GKE) to enable 2nd level cache Hibernate in Spring Boot application. Spingboot app connect to Spanner by using spanner-jdbc (https://github.com/olavloite/spanner-jdbc).
I deploy 1 service (replicas=2) to k8s, however hazelcast instances located in 2 pods cannot connect stably (it works when server starts but then seem disconnected after, please the below log). This causes 2nd level cache cannot be shared between 2 pods. Anyone can help this situation? Thank you.
<groupId>com.hazelcast.poc</groupId>
<artifactId>hazelcast-poc</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>hazelcast-poc</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>nl.topicus</groupId>
<artifactId>spanner-jdbc</artifactId>
<version>1.0.8</version>
</dependency>
<dependency>
<groupId>nl.topicus</groupId>
<artifactId>spanner-hibernate</artifactId>
<version>0.7</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-rest</artifactId>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast</artifactId>
<version>3.11</version>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast-hibernate5</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast-kubernetes</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.8.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.uuid</groupId>
<artifactId>java-uuid-generator</artifactId>
<version>3.1.5</version>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-java8</artifactId>
<version>${hibernate.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.0.1.RELEASE</version>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
</plugins>
</build>
Springboot Application.java:
public static void main(String args) throws IOException {
CloudSpannerDriver.setLogLevel(CloudSpannerDriver.DEBUG);
Writer writer = new FileWriter("jdbc.log", false);
DriverManager.setLogWriter(new PrintWriter(writer));SpringApplication.run(Application.class, args);
}
@Bean
public Config hazelCastConfig(){
Config config = new Config();
config
.addMapConfig(
new MapConfig()
.setName("employees")
.setMaxSizeConfig(new MaxSizeConfig(200, MaxSizeConfig.MaxSizePolicy.FREE_HEAP_SIZE))
.setEvictionPolicy(EvictionPolicy.LRU)
.setTimeToLiveSeconds(20000));
JoinConfig joinConfig = config.getNetworkConfig().getJoin();
joinConfig.getMulticastConfig().setEnabled(false);
joinConfig.getKubernetesConfig().setEnabled(true).setProperty("namespace", "default");
return config;
}
}
deployment.yaml:
apiVersion: apps/v1
kind: Deployment
metadata:
name: hazelcast-embedded-0
spec:
replicas: 2
selector:
matchLabels:
app: hazelcast-embedded-0
template:
metadata:
labels:
app: hazelcast-embedded-0
spec:
volumes:
- name: hazelcast-key
secret:
secretName: hazelcast-key
containers:
- name: hazelcast-embedded
image: gcr.io/testproject-221915/hazelcast- poc@sha256:5a103d7e2e56c46e9e42617232e472891bda5d121139fe7e41057596bd71bf00
imagePullPolicy: Always
volumeMounts:
- name: hazelcast-key
mountPath: /var/secrets/google
env:
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /var/secrets/google/key.json
ports:
- containerPort: 5701
- containerPort: 8080
apiVersion: v1
kind: Service
metadata:
name: hazelcast-embedded-0
spec:
type: LoadBalancer
selector:
app: hazelcast-embedded-0
ports:
- name: hazelcast
port: 5701
- name: app
port: 8080
Spanner Datasource file:
@Bean
public LocalContainerEntityManagerFactoryBean spannerEntityManager() {
LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean();
em.setDataSource(spannerDataSource());
em.setPersistenceUnitName("SpannerPU");
HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter();
em.setJpaVendorAdapter(vendorAdapter);
HashMap<String, Object> properties = new HashMap<>();
properties.put("hibernate.dialect", "nl.topicus.hibernate.dialect.CloudSpannerDialect");
properties.put("hibernate.temp.use_jdbc_metadata_defaults", false);
properties.put("hibernate.show_sql", showSql);
properties.put("hibernate.format_sql", formatSql);
properties.put("hibernate.cache.use_query_cache", true);
properties.put("hibernate.cache.use_second_level_cache", true);
properties.put("hibernate.cache.region.factory_class", "com.hazelcast.hibernate.HazelcastCacheRegionFactory");
if("development".equals(profileActive)) {
properties.put("hibernate.generate_statistics", statistic);
properties.put("org.hibernate.stat", hibernateStat);
}
em.setJpaPropertyMap(properties);
return em;
}
Application.properties
spanner.datasource.className=nl.topicus.jdbc.CloudSpannerDriver
spanner.datasource.url=jdbc:cloudspanner://;Project=testproject-
221915;Instance=test-hazelcast;Database=hazelcast
spanner.hibernate.dialect=
nl.topicus.hibernate.dialect.CloudSpannerDialect
spanner.hibernate.show-sql=true
spanner.hibernate.format_sql=true
spanner.datasource.hikari.maximum-pool-size=2 #depend on
microservice
spanner.datasource.hikari.minimum-idle=2
spanner.datasource.hikari.connection-timeout=60000
spring.profiles.active=dev
hibernate.generate_statistics=true
org.hibernate.stat=DEBUG
Log_pod1
Log_pod2
As you see, the second log has only 1 member whereas the first has 2 members.
I tested by issuing rest request to service which then distributed to each pods, the test result is sometimes cache are shared between two pods, sometimes not.
I expected 2nd Hibernate cache should be shared between 2 pods in the service
hibernate spring-boot kubernetes hazelcast
Iam using embedded Hazelcast deployed in Kubernetes(GKE) to enable 2nd level cache Hibernate in Spring Boot application. Spingboot app connect to Spanner by using spanner-jdbc (https://github.com/olavloite/spanner-jdbc).
I deploy 1 service (replicas=2) to k8s, however hazelcast instances located in 2 pods cannot connect stably (it works when server starts but then seem disconnected after, please the below log). This causes 2nd level cache cannot be shared between 2 pods. Anyone can help this situation? Thank you.
<groupId>com.hazelcast.poc</groupId>
<artifactId>hazelcast-poc</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>hazelcast-poc</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>nl.topicus</groupId>
<artifactId>spanner-jdbc</artifactId>
<version>1.0.8</version>
</dependency>
<dependency>
<groupId>nl.topicus</groupId>
<artifactId>spanner-hibernate</artifactId>
<version>0.7</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-rest</artifactId>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast</artifactId>
<version>3.11</version>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast-hibernate5</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast-kubernetes</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.8.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.uuid</groupId>
<artifactId>java-uuid-generator</artifactId>
<version>3.1.5</version>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-java8</artifactId>
<version>${hibernate.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.0.1.RELEASE</version>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
</plugins>
</build>
Springboot Application.java:
public static void main(String args) throws IOException {
CloudSpannerDriver.setLogLevel(CloudSpannerDriver.DEBUG);
Writer writer = new FileWriter("jdbc.log", false);
DriverManager.setLogWriter(new PrintWriter(writer));SpringApplication.run(Application.class, args);
}
@Bean
public Config hazelCastConfig(){
Config config = new Config();
config
.addMapConfig(
new MapConfig()
.setName("employees")
.setMaxSizeConfig(new MaxSizeConfig(200, MaxSizeConfig.MaxSizePolicy.FREE_HEAP_SIZE))
.setEvictionPolicy(EvictionPolicy.LRU)
.setTimeToLiveSeconds(20000));
JoinConfig joinConfig = config.getNetworkConfig().getJoin();
joinConfig.getMulticastConfig().setEnabled(false);
joinConfig.getKubernetesConfig().setEnabled(true).setProperty("namespace", "default");
return config;
}
}
deployment.yaml:
apiVersion: apps/v1
kind: Deployment
metadata:
name: hazelcast-embedded-0
spec:
replicas: 2
selector:
matchLabels:
app: hazelcast-embedded-0
template:
metadata:
labels:
app: hazelcast-embedded-0
spec:
volumes:
- name: hazelcast-key
secret:
secretName: hazelcast-key
containers:
- name: hazelcast-embedded
image: gcr.io/testproject-221915/hazelcast- poc@sha256:5a103d7e2e56c46e9e42617232e472891bda5d121139fe7e41057596bd71bf00
imagePullPolicy: Always
volumeMounts:
- name: hazelcast-key
mountPath: /var/secrets/google
env:
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /var/secrets/google/key.json
ports:
- containerPort: 5701
- containerPort: 8080
apiVersion: v1
kind: Service
metadata:
name: hazelcast-embedded-0
spec:
type: LoadBalancer
selector:
app: hazelcast-embedded-0
ports:
- name: hazelcast
port: 5701
- name: app
port: 8080
Spanner Datasource file:
@Bean
public LocalContainerEntityManagerFactoryBean spannerEntityManager() {
LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean();
em.setDataSource(spannerDataSource());
em.setPersistenceUnitName("SpannerPU");
HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter();
em.setJpaVendorAdapter(vendorAdapter);
HashMap<String, Object> properties = new HashMap<>();
properties.put("hibernate.dialect", "nl.topicus.hibernate.dialect.CloudSpannerDialect");
properties.put("hibernate.temp.use_jdbc_metadata_defaults", false);
properties.put("hibernate.show_sql", showSql);
properties.put("hibernate.format_sql", formatSql);
properties.put("hibernate.cache.use_query_cache", true);
properties.put("hibernate.cache.use_second_level_cache", true);
properties.put("hibernate.cache.region.factory_class", "com.hazelcast.hibernate.HazelcastCacheRegionFactory");
if("development".equals(profileActive)) {
properties.put("hibernate.generate_statistics", statistic);
properties.put("org.hibernate.stat", hibernateStat);
}
em.setJpaPropertyMap(properties);
return em;
}
Application.properties
spanner.datasource.className=nl.topicus.jdbc.CloudSpannerDriver
spanner.datasource.url=jdbc:cloudspanner://;Project=testproject-
221915;Instance=test-hazelcast;Database=hazelcast
spanner.hibernate.dialect=
nl.topicus.hibernate.dialect.CloudSpannerDialect
spanner.hibernate.show-sql=true
spanner.hibernate.format_sql=true
spanner.datasource.hikari.maximum-pool-size=2 #depend on
microservice
spanner.datasource.hikari.minimum-idle=2
spanner.datasource.hikari.connection-timeout=60000
spring.profiles.active=dev
hibernate.generate_statistics=true
org.hibernate.stat=DEBUG
Log_pod1
Log_pod2
As you see, the second log has only 1 member whereas the first has 2 members.
I tested by issuing rest request to service which then distributed to each pods, the test result is sometimes cache are shared between two pods, sometimes not.
I expected 2nd Hibernate cache should be shared between 2 pods in the service
hibernate spring-boot kubernetes hazelcast
hibernate spring-boot kubernetes hazelcast
edited Jan 4 at 17:18
Tom Ho
asked Jan 4 at 10:33
Tom HoTom Ho
62
62
Can you share the complete log (without| grep Member
)?
– Rafał Leszko
Jan 4 at 15:53
Hi @Rafał Leszko, I just updated the post with log of pod1 and pod2. I am trying to translate a line in log "Config seed port is 5701 and cluster size is 1. Some of the ports seem occupied!". Do you have any idea ?
– Tom Ho
Jan 4 at 16:35
add a comment |
Can you share the complete log (without| grep Member
)?
– Rafał Leszko
Jan 4 at 15:53
Hi @Rafał Leszko, I just updated the post with log of pod1 and pod2. I am trying to translate a line in log "Config seed port is 5701 and cluster size is 1. Some of the ports seem occupied!". Do you have any idea ?
– Tom Ho
Jan 4 at 16:35
Can you share the complete log (without
| grep Member
)?– Rafał Leszko
Jan 4 at 15:53
Can you share the complete log (without
| grep Member
)?– Rafał Leszko
Jan 4 at 15:53
Hi @Rafał Leszko, I just updated the post with log of pod1 and pod2. I am trying to translate a line in log "Config seed port is 5701 and cluster size is 1. Some of the ports seem occupied!". Do you have any idea ?
– Tom Ho
Jan 4 at 16:35
Hi @Rafał Leszko, I just updated the post with log of pod1 and pod2. I am trying to translate a line in log "Config seed port is 5701 and cluster size is 1. Some of the ports seem occupied!". Do you have any idea ?
– Tom Ho
Jan 4 at 16:35
add a comment |
1 Answer
1
active
oldest
votes
You are creating two Hazelcast instances
At 9:36:44.990 a kubernetes discovery instance
At 9:36:53.706 a multicast discovery instance
Might be this again
Set hibernate.cache.hazelcast.instance_name
system property and config.setInstanceName("whatever")
pairing should match
Thanks, if we set so then both pods would use the same hazelcast instance. Could you explain why different hazelcast instances for different pods causes the issue. Also, why need to set both system properties and in code?
– Tom Ho
Jan 5 at 3:40
@TomHo with your config, your Spring-Boot app creates 2 instances in the same application, one that you configured & one for the Hibernate. Parameter Neil mentioned is to inform hazelcast-hibernate module not to create a separate instance but use the already created one. If you have 2 pods, both will still create their own instances, but only using Kubernetes Discovery.
– Gokhan Oner
Jan 5 at 7:01
@GokhanOner, thanks for explanation, it works now.
– Tom Ho
Jan 10 at 15:58
add a comment |
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f54037219%2fhazelcast-instance-used-for-2nd-hibernate-cache-cannot-connect-stably-in-kuberne%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
1 Answer
1
active
oldest
votes
1 Answer
1
active
oldest
votes
active
oldest
votes
active
oldest
votes
You are creating two Hazelcast instances
At 9:36:44.990 a kubernetes discovery instance
At 9:36:53.706 a multicast discovery instance
Might be this again
Set hibernate.cache.hazelcast.instance_name
system property and config.setInstanceName("whatever")
pairing should match
Thanks, if we set so then both pods would use the same hazelcast instance. Could you explain why different hazelcast instances for different pods causes the issue. Also, why need to set both system properties and in code?
– Tom Ho
Jan 5 at 3:40
@TomHo with your config, your Spring-Boot app creates 2 instances in the same application, one that you configured & one for the Hibernate. Parameter Neil mentioned is to inform hazelcast-hibernate module not to create a separate instance but use the already created one. If you have 2 pods, both will still create their own instances, but only using Kubernetes Discovery.
– Gokhan Oner
Jan 5 at 7:01
@GokhanOner, thanks for explanation, it works now.
– Tom Ho
Jan 10 at 15:58
add a comment |
You are creating two Hazelcast instances
At 9:36:44.990 a kubernetes discovery instance
At 9:36:53.706 a multicast discovery instance
Might be this again
Set hibernate.cache.hazelcast.instance_name
system property and config.setInstanceName("whatever")
pairing should match
Thanks, if we set so then both pods would use the same hazelcast instance. Could you explain why different hazelcast instances for different pods causes the issue. Also, why need to set both system properties and in code?
– Tom Ho
Jan 5 at 3:40
@TomHo with your config, your Spring-Boot app creates 2 instances in the same application, one that you configured & one for the Hibernate. Parameter Neil mentioned is to inform hazelcast-hibernate module not to create a separate instance but use the already created one. If you have 2 pods, both will still create their own instances, but only using Kubernetes Discovery.
– Gokhan Oner
Jan 5 at 7:01
@GokhanOner, thanks for explanation, it works now.
– Tom Ho
Jan 10 at 15:58
add a comment |
You are creating two Hazelcast instances
At 9:36:44.990 a kubernetes discovery instance
At 9:36:53.706 a multicast discovery instance
Might be this again
Set hibernate.cache.hazelcast.instance_name
system property and config.setInstanceName("whatever")
pairing should match
You are creating two Hazelcast instances
At 9:36:44.990 a kubernetes discovery instance
At 9:36:53.706 a multicast discovery instance
Might be this again
Set hibernate.cache.hazelcast.instance_name
system property and config.setInstanceName("whatever")
pairing should match
answered Jan 4 at 16:52
Neil StevensonNeil Stevenson
1,527410
1,527410
Thanks, if we set so then both pods would use the same hazelcast instance. Could you explain why different hazelcast instances for different pods causes the issue. Also, why need to set both system properties and in code?
– Tom Ho
Jan 5 at 3:40
@TomHo with your config, your Spring-Boot app creates 2 instances in the same application, one that you configured & one for the Hibernate. Parameter Neil mentioned is to inform hazelcast-hibernate module not to create a separate instance but use the already created one. If you have 2 pods, both will still create their own instances, but only using Kubernetes Discovery.
– Gokhan Oner
Jan 5 at 7:01
@GokhanOner, thanks for explanation, it works now.
– Tom Ho
Jan 10 at 15:58
add a comment |
Thanks, if we set so then both pods would use the same hazelcast instance. Could you explain why different hazelcast instances for different pods causes the issue. Also, why need to set both system properties and in code?
– Tom Ho
Jan 5 at 3:40
@TomHo with your config, your Spring-Boot app creates 2 instances in the same application, one that you configured & one for the Hibernate. Parameter Neil mentioned is to inform hazelcast-hibernate module not to create a separate instance but use the already created one. If you have 2 pods, both will still create their own instances, but only using Kubernetes Discovery.
– Gokhan Oner
Jan 5 at 7:01
@GokhanOner, thanks for explanation, it works now.
– Tom Ho
Jan 10 at 15:58
Thanks, if we set so then both pods would use the same hazelcast instance. Could you explain why different hazelcast instances for different pods causes the issue. Also, why need to set both system properties and in code?
– Tom Ho
Jan 5 at 3:40
Thanks, if we set so then both pods would use the same hazelcast instance. Could you explain why different hazelcast instances for different pods causes the issue. Also, why need to set both system properties and in code?
– Tom Ho
Jan 5 at 3:40
@TomHo with your config, your Spring-Boot app creates 2 instances in the same application, one that you configured & one for the Hibernate. Parameter Neil mentioned is to inform hazelcast-hibernate module not to create a separate instance but use the already created one. If you have 2 pods, both will still create their own instances, but only using Kubernetes Discovery.
– Gokhan Oner
Jan 5 at 7:01
@TomHo with your config, your Spring-Boot app creates 2 instances in the same application, one that you configured & one for the Hibernate. Parameter Neil mentioned is to inform hazelcast-hibernate module not to create a separate instance but use the already created one. If you have 2 pods, both will still create their own instances, but only using Kubernetes Discovery.
– Gokhan Oner
Jan 5 at 7:01
@GokhanOner, thanks for explanation, it works now.
– Tom Ho
Jan 10 at 15:58
@GokhanOner, thanks for explanation, it works now.
– Tom Ho
Jan 10 at 15:58
add a comment |
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f54037219%2fhazelcast-instance-used-for-2nd-hibernate-cache-cannot-connect-stably-in-kuberne%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Can you share the complete log (without
| grep Member
)?– Rafał Leszko
Jan 4 at 15:53
Hi @Rafał Leszko, I just updated the post with log of pod1 and pod2. I am trying to translate a line in log "Config seed port is 5701 and cluster size is 1. Some of the ports seem occupied!". Do you have any idea ?
– Tom Ho
Jan 4 at 16:35