Bdtjtk

Iam using embedded Hazelcast deployed in Kubernetes(GKE) to enable 2nd level cache Hibernate in Spring Boot application. Spingboot app connect to Spanner by using spanner-jdbc (https://github.com/olavloite/spanner-jdbc).
I deploy 1 service (replicas=2) to k8s, however hazelcast instances located in 2 pods cannot connect stably (it works when server starts but then seem disconnected after, please the below log). This causes 2nd level cache cannot be shared between 2 pods. Anyone can help this situation? Thank you.

<groupId>com.hazelcast.poc</groupId>

<artifactId>hazelcast-poc</artifactId>

<version>0.0.1-SNAPSHOT</version>

<name>hazelcast-poc</name>

<description>Demo project for Spring Boot</description>



<properties>

    <java.version>1.8</java.version>

</properties>



<dependencies>

    <dependency>

        <groupId>nl.topicus</groupId>

        <artifactId>spanner-jdbc</artifactId>

        <version>1.0.8</version>

    </dependency>



    <dependency>

        <groupId>nl.topicus</groupId>

        <artifactId>spanner-hibernate</artifactId>

        <version>0.7</version>

    </dependency>

    <dependency>

        <groupId>org.springframework.boot</groupId>

        <artifactId>spring-boot-starter-data-jpa</artifactId>

    </dependency>

    <dependency>

        <groupId>org.springframework.boot</groupId>

        <artifactId>spring-boot-starter-data-rest</artifactId>

    </dependency>

    <dependency>

        <groupId>com.hazelcast</groupId>

        <artifactId>hazelcast</artifactId>

        <version>3.11</version>

    </dependency>

    <dependency>

        <groupId>com.hazelcast</groupId>

        <artifactId>hazelcast-hibernate5</artifactId>

        <version>1.3.0</version>

    </dependency>

    <dependency>

        <groupId>com.hazelcast</groupId>

        <artifactId>hazelcast-kubernetes</artifactId>

        <version>1.3.1</version>

    </dependency>

    <dependency>

        <groupId>org.springframework.boot</groupId>

        <artifactId>spring-boot-starter-test</artifactId>

        <scope>test</scope>

    </dependency>



    <dependency>

        <groupId>org.assertj</groupId>

        <artifactId>assertj-core</artifactId>

        <version>3.8.0</version>

        <scope>test</scope>

    </dependency>

    <dependency>

        <groupId>org.hamcrest</groupId>

        <artifactId>hamcrest-all</artifactId>

        <version>1.3</version>

        <scope>test</scope>

    </dependency>

    <dependency>

        <groupId>com.fasterxml.uuid</groupId>

        <artifactId>java-uuid-generator</artifactId>

        <version>3.1.5</version>

    </dependency>

    <dependency>

        <groupId>org.hibernate</groupId>

        <artifactId>hibernate-java8</artifactId>

        <version>${hibernate.version}</version>

    </dependency>

    <dependency>

        <groupId>junit</groupId>

        <artifactId>junit</artifactId>

        <scope>test</scope>

    </dependency>

</dependencies>



<build>

    <plugins>

        <plugin>

            <groupId>org.springframework.boot</groupId>

            <artifactId>spring-boot-maven-plugin</artifactId>

            <version>2.0.1.RELEASE</version>

            <executions>

                <execution>

                    <goals>

                        <goal>repackage</goal>

                    </goals>

                </execution>

            </executions>

        </plugin>

        <plugin>

            <groupId>org.apache.maven.plugins</groupId>

            <artifactId>maven-compiler-plugin</artifactId>

            <version>3.8.0</version>

            <configuration>

                <source>1.8</source>

                <target>1.8</target>

                <encoding>UTF-8</encoding>

            </configuration>

        </plugin>

    </plugins>

</build>

Springboot Application.java:

public static void main(String args) throws IOException {

    CloudSpannerDriver.setLogLevel(CloudSpannerDriver.DEBUG);

    Writer writer = new FileWriter("jdbc.log", false);

    DriverManager.setLogWriter(new PrintWriter(writer));SpringApplication.run(Application.class, args);

}



@Bean

public Config hazelCastConfig(){

    Config config = new Config();

    config

            .addMapConfig(

                    new MapConfig()

                            .setName("employees")

                            .setMaxSizeConfig(new MaxSizeConfig(200, MaxSizeConfig.MaxSizePolicy.FREE_HEAP_SIZE))

                            .setEvictionPolicy(EvictionPolicy.LRU)

                            .setTimeToLiveSeconds(20000));

    JoinConfig joinConfig = config.getNetworkConfig().getJoin();

    joinConfig.getMulticastConfig().setEnabled(false);

    joinConfig.getKubernetesConfig().setEnabled(true).setProperty("namespace", "default");



    return config;

}

}

deployment.yaml:

apiVersion: apps/v1

kind: Deployment

metadata:

 name: hazelcast-embedded-0

spec:

  replicas: 2

  selector:

  matchLabels:

  app: hazelcast-embedded-0

  template:

   metadata:

    labels:

      app: hazelcast-embedded-0

    spec:

    volumes:

      - name: hazelcast-key

        secret:

        secretName: hazelcast-key

  containers:

    - name: hazelcast-embedded

      image: gcr.io/testproject-221915/hazelcast- poc@sha256:5a103d7e2e56c46e9e42617232e472891bda5d121139fe7e41057596bd71bf00

      imagePullPolicy: Always

      volumeMounts:

        - name: hazelcast-key

          mountPath: /var/secrets/google

      env:

      - name: GOOGLE_APPLICATION_CREDENTIALS

        value: /var/secrets/google/key.json

      ports:

       - containerPort: 5701

       - containerPort: 8080





  apiVersion: v1

  kind: Service

  metadata:

  name: hazelcast-embedded-0

  spec:

   type: LoadBalancer

   selector:

    app: hazelcast-embedded-0

   ports:

   - name: hazelcast

    port: 5701

   - name: app

     port: 8080

Spanner Datasource file:

@Bean

public LocalContainerEntityManagerFactoryBean spannerEntityManager() {

    LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean();

    em.setDataSource(spannerDataSource());

    em.setPersistenceUnitName("SpannerPU");

    HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter();

    em.setJpaVendorAdapter(vendorAdapter);

    HashMap<String, Object> properties = new HashMap<>();

    properties.put("hibernate.dialect", "nl.topicus.hibernate.dialect.CloudSpannerDialect");

    properties.put("hibernate.temp.use_jdbc_metadata_defaults", false);

    properties.put("hibernate.show_sql", showSql);

    properties.put("hibernate.format_sql", formatSql);

    properties.put("hibernate.cache.use_query_cache", true);

    properties.put("hibernate.cache.use_second_level_cache", true);

    properties.put("hibernate.cache.region.factory_class", "com.hazelcast.hibernate.HazelcastCacheRegionFactory");

    if("development".equals(profileActive)) {

        properties.put("hibernate.generate_statistics", statistic);

        properties.put("org.hibernate.stat", hibernateStat);            

    }        

    em.setJpaPropertyMap(properties);



    return em;

}

Application.properties

spanner.datasource.className=nl.topicus.jdbc.CloudSpannerDriver

spanner.datasource.url=jdbc:cloudspanner://;Project=testproject- 

221915;Instance=test-hazelcast;Database=hazelcast

spanner.hibernate.dialect=

 nl.topicus.hibernate.dialect.CloudSpannerDialect

spanner.hibernate.show-sql=true

spanner.hibernate.format_sql=true

spanner.datasource.hikari.maximum-pool-size=2 #depend on 

 microservice

spanner.datasource.hikari.minimum-idle=2

spanner.datasource.hikari.connection-timeout=60000



spring.profiles.active=dev

hibernate.generate_statistics=true

org.hibernate.stat=DEBUG

Log_pod1
Log_pod2

As you see, the second log has only 1 member whereas the first has 2 members.
I tested by issuing rest request to service which then distributed to each pods, the test result is sometimes cache are shared between two pods, sometimes not.

I expected 2nd Hibernate cache should be shared between 2 pods in the service

You are creating two Hazelcast instances

At 9:36:44.990 a kubernetes discovery instance

At 9:36:53.706 a multicast discovery instance

Might be this again

Set hibernate.cache.hazelcast.instance_name system property and config.setInstanceName("whatever") pairing should match