Option Elasticsearch in jhipster 7.1.0 is not working

dunfe commented 3 years ago

Overview of the issue

Search function is not working.

Motivation for or Use Case

I'm creating an application with Jhipster stack, I want to search items on some entities, then I turn on option Elasticsearch. The project is generated without any issue. But when I try to search in entity admin page then the result always is empty No Products found although I tried to test all item properties.

Reproduce the error

Install jhipster global: npm install -g generator-jhipster
Make project dir, copy jdl file into this folder then run: jhipster jdl ./app.jdl
Start back-end: ./gradlew -x webapp
Start front-end: npm start
Go to: localhost:9000
Login with admin account
Choose Product from entities menu
Type a product name into search input
Press enter

JHipster Version(s)

7.1.0

JHipster configuration, a `.yo-rc.json` file generated in the root folder

.yo-rc.json file

{
  "generator-jhipster": {
    "authenticationType": "jwt",
    "cacheProvider": "redis",
    "clientFramework": "react",
    "serverPort": "8080",
    "serviceDiscoveryType": false,
    "skipUserManagement": false,
    "withAdminUi": true,
    "baseName": "nashop",
    "buildTool": "gradle",
    "databaseType": "sql",
    "devDatabaseType": "mysql",
    "enableHibernateCache": true,
    "enableSwaggerCodegen": false,
    "enableTranslation": true,
    "jhiPrefix": "jhi",
    "languages": ["vi", "en"],
    "messageBroker": false,
    "prodDatabaseType": "mysql",
    "searchEngine": "elasticsearch",
    "skipClient": false,
    "testFrameworks": [],
    "websocket": false,
    "packageName": "com.na.shop",
    "packageFolder": "com/na/shop",
    "applicationType": "monolith",
    "nativeLanguage": "vi",
    "jhipsterVersion": "7.1.0",
    "skipServer": false,
    "clientPackageManager": "npm",
    "dtoSuffix": "DTO",
    "entitySuffix": "",
    "reactive": false,
    "clientTheme": "none",
    "clientThemeVariant": "",
    "applicationIndex": 0,
    "entities": [
      "Address",
      "Category",
      "Kind",
      "DonHang",
      "DonHangItems",
      "Player",
      "Position",
      "Product",
      "ProductImage",
      "Sport",
      "Team",
      "PaymentMethod"
    ],
    "skipCheckLengthOfIdentifier": false,
    "skipFakeData": false,
    "blueprints": [],
    "otherModules": [],
    "pages": [],
    "creationTimestamp": 1629698069843,
    "jwtSecretKey": "YourJWTSecretKeyWasReplacedByThisMeaninglessTextByTheJHipsterInfoCommandForObviousSecurityReasons",
    "devServerPort": 9060,
    "lastLiquibaseTimestamp": 1629698789000
  }
}

JDL for the Entity configuration(s) `entityName.json` files generated in the `.jhipster` directory

JDL entity definitions

application {
  config {
    baseName nashop
    packageName com.na.shop
    applicationType monolith
    authenticationType jwt
    buildTool gradle
    cacheProvider redis
    clientFramework react
    databaseType sql
    devDatabaseType mysql
    prodDatabaseType mysql
    languages [vi, en]
    nativeLanguage vi
    searchEngine elasticsearch
  }

  entities *
}

entity Address {
  addressLine String required
  country String required
  province String required
  phone String required
}
entity Category {
  name String required
  imageUrl String pattern(/(https?:\/\/\S+(?:png|jpe?g|gif)\S*)/)
  createAt Instant
}
entity Kind {
  name String required
  imageUrl String pattern(/(https?:\/\/\S+(?:png|jpe?g|gif)\S*)/)
  createAt Instant
}
entity DonHang {
  status DonHangStatus required
  createAt Instant
  modifiedAt Instant
  total Float required
  note String
}
entity DonHangItems {
  quantity Integer required
  createAt Instant
  modifiedAt Instant
}
entity Player {
  name String required
  imageUrl String pattern(/(https?:\/\/\S+(?:png|jpe?g|gif)\S*)/)
  createAt Instant
}
entity Position {
  name String required
  createAt Instant
}
entity Product {
  name String required
  imageUrl String pattern(/(https?:\/\/\S+(?:png|jpe?g|gif)\S*)/)
  price Float required
  description String
  inventory Integer required
  status ProductStatus required
  createAt Instant
}
entity ProductImage {
  imageUrl String required pattern(/(https?:\/\/\S+(?:png|jpe?g|gif)\S*)/)
  createAt Instant
}
entity Sport {
  name String required
  imageUrl String pattern(/(https?:\/\/\S+(?:png|jpe?g|gif)\S*)/)
  createAt Instant
}
entity Team {
  name String required
  imageUrl String pattern(/(https?:\/\/\S+(?:png|jpe?g|gif)\S*)/)
  createAt Instant
}
entity PaymentMethod {
  name String required
  description String
  imageUrl String pattern(/(https?:\/\/\S+(?:png|jpe?g|gif)\S*)/)
  createAt Instant
}
enum DonHangStatus {
  CANCELED,
  DELIVERED,
  INTRANSIT,
  PAYMENTDUE,
  PROBLEM,
  PROCESSING,
  RETURNED
}
enum ProductStatus {
  AVAILABLE,
  UNAVAILABLE
}

relationship OneToOne {
  Address{user} to User
}
relationship OneToMany {
  PaymentMethod{donHang} to DonHang{paymentMethod(name) required}
  DonHang{donHangItems} to DonHangItems{donhang required}
  Product{donHangItems} to DonHangItems{product(name) required}
  Position{player} to Player{position(name) required}
  Team{player} to Player{team(name) required}
  Category{product} to Product{category(name) required}
  Player{product} to Product{player(name)}
  Product{productImage} to ProductImage{product(name) required}
}
relationship ManyToOne {
  DonHang{user required} to User
  Team{sport(name)} to Sport
}
relationship ManyToMany {
  DonHangItems{accessory(name)} to Product{donhangItems}
  Product{kind(name)} to Kind{product(name)}
}

search DonHang, Player, Product, Team with elasticsearch

Environment and Tools

openjdk version "11.0.12" 2021-07-20 OpenJDK Runtime Environment Temurin-11.0.12+7 (build 11.0.12+7) OpenJDK 64-Bit Server VM Temurin-11.0.12+7 (build 11.0.12+7, mixed mode)

git version 2.24.1.windows.2

node: v14.17.5

npm: 6.14.14

Docker version 20.10.8, build 3967b7d

[x] Checking this box is mandatory (this is just to show you read everything)

mraible commented 3 years ago

Did you start Elasticsearch in a Docker container?

docker-compose -f src/main/docker/elasticsearch.yml up

dunfe commented 3 years ago

Did you start Elasticsearch in a Docker container?
docker-compose -f src/main/docker/elasticsearch.yml up

Absolute yes, my project is start normaly, this is debug log when I press search button

Debug log

2021-08-22 13:38:59.948 DEBUG 18664 --- [  XNIO-1 task-2] com.na.shop.service.ProductService       : Request to search for a page of Products for query Steel
2021-08-22 13:38:59.958 DEBUG 18664 --- [  XNIO-1 task-2] com.na.shop.service.ProductService       : Exit: search() with result = Page 1 of 0 containing UNKNOWN instances
2021-08-22 13:38:59.961 DEBUG 18664 --- [  XNIO-1 task-2] com.na.shop.web.rest.ProductResource     : Exit: searchProducts() with result = <200 OK OK,[],[X-Total-Count:"0", Link:"; rel="last",; rel="first""]>

mshima commented 3 years ago

https://github.com/jhipster/generator-jhipster/issues/11400? If you create a new registry and look for it, it works?

dunfe commented 3 years ago

11400?

If you create a new registry and look for it, it works?

It doesn't work, too, I turn off faker then put my own data already. I tried some query like query=card query=name:card but still empty response although the Category with name='Card' existed.

Debug log

2021-08-24 09:33:24.287 DEBUG 19452 --- [  XNIO-1 task-2] com.na.shop.web.rest.CategoryResource    : Exit: getAllCategories() with result = [Category{id=1, name='Card', imageUrl='null', createAt='2021-08-16T00:00:00Z'}, Category{id=2, name='Repack', imageUrl='null', createAt='2021-08-16T00:00:00Z'}, Category{id=3, name='Box', imageUrl='null', createAt='2021-08-16T00:00:00Z'}, Category{id=4, name='Ph? ki?n', imageUrl='null', createAt='2021-08-16T00:00:00Z'}, Category{id=5, name='??c bi?t', imageUrl='null', createAt='2021-08-17T00:00:00Z'}]
2021-08-24 09:33:28.567 DEBUG 19452 --- [  XNIO-1 task-2] com.na.shop.web.rest.CategoryResource    : Enter: searchCategories() with argument[s] = [name:card]
2021-08-24 09:33:28.567 DEBUG 19452 --- [  XNIO-1 task-2] com.na.shop.web.rest.CategoryResource    : REST request to search Categories for query name:card
2021-08-24 09:33:28.573 DEBUG 19452 --- [  XNIO-1 task-2] com.na.shop.web.rest.CategoryResource    : Exit: searchCategories() with result = []

pascalgrimaud commented 2 years ago

@dungreact : if you search an entity which was already generated by faker.js, it's normal as it is not indexed by elasticsearch. Your use case should be:

add a new entry
search it

Can you try it plz?

dunfe commented 2 years ago

@dungreact : if you search an entity which was already generated by faker.js, it's normal as it is not indexed by elasticsearch. Your use case should be:

add a new entry

search it

Can you try it plz?

Hi, I already have my own data. I think the problem is they are not indexed. I tried to use Elasticsearch Reindexer but it have alot of error.

github-actions[bot] commented 2 years ago

This issue is stale because it has been open 30 days with no activity. Our core developers tend to be more verbose on denying. If there is no negative comment, possibly this feature will be accepted. We are accepting PRs :smiley:. Comment or this will be closed in 7 days

Tcharl commented 2 years ago

You should call the service.save() method in order to get something that is indexed, it's not really related to faker or other.

I had something that worked with reeindexer, but didn't know where are the source.

here's a snippet for some dedicated repositories:

package com.thalesgroup.tce.service.marketplace.service;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.thalesgroup.tce.service.marketplace.domain.ApplicationDefinition;
import com.pkg.domain.ApplicationDefinitionComment;
import com.pkg.domain.ApplicationDefinitionTag;
import com.pkg.service.marketplace.domain.User;
import com.pkg.service.marketplace.repository.ApplicationDefinitionCommentRepository;
import com.pkg.service.marketplace.repository.ApplicationDefinitionRepository;
import com.pkg.service.marketplace.repository.ApplicationDefinitionTagRepository;
import com.pkg.service.marketplace.repository.UserRepository;
import com.pkg.service.marketplace.repository.search.ApplicationDefinitionCommentSearchRepository;
import com.pkg.service.marketplace.repository.search.ApplicationDefinitionSearchRepository;
import com.pkg.service.marketplace.repository.search.ApplicationDefinitionTagSearchRepository;
import com.pkg.service.marketplace.repository.search.UserSearchRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Scope;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import javax.persistence.ManyToMany;
import java.beans.IntrospectionException;
import java.beans.PropertyDescriptor;
import java.io.Serializable;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;

import static org.springframework.beans.factory.config.ConfigurableBeanFactory.SCOPE_SINGLETON;

@SuppressWarnings({"squid:S00107", "squid:CallToDeprecatedMethod", "squid:S00119"})
@Service
@Transactional(readOnly = true)
@ConditionalOnProperty("spring.data.jest.uri")
@Scope(SCOPE_SINGLETON)
public class ElasticsearchIndexService {

    private final Logger log = LoggerFactory.getLogger(ElasticsearchIndexService.class);

    private final ApplicationDefinitionSearchRepository applicationDefinitionSearchRepository;
    private final ApplicationDefinitionRepository applicationDefinitionRepository;

    private final ApplicationDefinitionCommentSearchRepository applicationDefinitionCommentSearchRepository;
    private final ApplicationDefinitionCommentRepository applicationDefinitionCommentRepository;

    private final ApplicationDefinitionTagSearchRepository applicationDefinitionTagSearchRepository;
    private final ApplicationDefinitionTagRepository applicationDefinitionTagRepository;

    private final UserSearchRepository userSearchRepository;
    private final UserRepository userRepository;

    private final ElasticsearchOperations elasticsearchTemplate;

    private static final Lock reindexLock = new ReentrantLock();

    /* JHIPSTER GENERATED CODE GIVES FOLLOWING SONAR WARNING:
     * Constructor has 9 parameters, which is greater than 7 authorized.
     */
    public ElasticsearchIndexService(
        ApplicationDefinitionSearchRepository applicationDefinitionSearchRepository,
        ApplicationDefinitionRepository applicationDefinitionRepository,
        ApplicationDefinitionCommentSearchRepository applicationDefinitionCommentSearchRepository,
        ApplicationDefinitionCommentRepository applicationDefinitionCommentRepository,
        ApplicationDefinitionTagSearchRepository applicationDefinitionTagSearchRepository,
        ApplicationDefinitionTagRepository applicationDefinitionTagRepository,
        UserSearchRepository userSearchRepository,
            UserRepository userRepository,

        ElasticsearchOperations elasticsearchTemplate) {
        this.applicationDefinitionSearchRepository = applicationDefinitionSearchRepository;
        this.applicationDefinitionRepository = applicationDefinitionRepository;
        this.applicationDefinitionCommentSearchRepository = applicationDefinitionCommentSearchRepository;
        this.applicationDefinitionCommentRepository = applicationDefinitionCommentRepository;
        this.applicationDefinitionTagSearchRepository = applicationDefinitionTagSearchRepository;
        this.applicationDefinitionTagRepository = applicationDefinitionTagRepository;
        this.userSearchRepository = userSearchRepository;
        this.userRepository = userRepository;

        this.elasticsearchTemplate = elasticsearchTemplate;
    }

    @Async
    @EventListener
    public void onApplicationEvent(ContextRefreshedEvent event) {
        this.reindexAll();
    }

    @Async
    public void reindexAll() {
        if(reindexLock.tryLock()) {
            try {
                reindexForClass(ApplicationDefinition.class, applicationDefinitionRepository, applicationDefinitionSearchRepository);
                reindexForClass(ApplicationDefinitionComment.class, applicationDefinitionCommentRepository, applicationDefinitionCommentSearchRepository);
                reindexForClass(ApplicationDefinitionTag.class, applicationDefinitionTagRepository, applicationDefinitionTagSearchRepository);
                reindexForClass(User.class, userRepository, userSearchRepository);

                // TODO reindex all model classes and implement this reindex on all microservices
                log.info("Elasticsearch: Successfully performed reindexing");
            } finally {
                reindexLock.unlock();
            }
        } else {
            log.info("Elasticsearch: concurrent reindexing attempt");
        }
    }

    @SuppressWarnings("unchecked")
    private <T, ID extends Serializable> void reindexForClass(Class<T> entityClass, JpaRepository<T, ID> jpaRepository,
                                                              ElasticsearchRepository<T, ID> elasticsearchRepository) {
        elasticsearchTemplate.deleteIndex(entityClass);
        elasticsearchTemplate.createIndex(entityClass);
        elasticsearchTemplate.putMapping(entityClass);
        if (jpaRepository.count() > 0) {
            // if a JHipster entity field is the owner side of a many-to-many relationship, it should be loaded manually
            List<Method> relationshipGetters = Arrays.stream(entityClass.getDeclaredFields())
                .filter(field -> field.getType().equals(Set.class))
                .filter(field -> field.getAnnotation(ManyToMany.class) != null)
                .filter(field -> field.getAnnotation(ManyToMany.class).mappedBy().isEmpty())
                .filter(field -> field.getAnnotation(JsonIgnore.class) == null)
                .map(field -> {
                    try {
                        return new PropertyDescriptor(field.getName(), entityClass).getReadMethod();
                    } catch (IntrospectionException e) {
                        log.error("Error retrieving getter for class {}, field {}. Field will NOT be indexed",
                            entityClass.getSimpleName(), field.getName(), e);
                        return null;
                    }
                })
                .filter(Objects::nonNull)
                .collect(Collectors.toList());

            int size = 100;
            for (int i = 0; i <= jpaRepository.count() / size; i++) {
                /* JHIPSTER GENERATED CODE GIVES FOLLOWING SONAR WARNING:
                 * Remove this use of "PageRequest"; it is deprecated
                 */
                Pageable page = PageRequest.of(i, size);
                log.info("Indexing page {} of {}, size {}", i, jpaRepository.count() / size, size);
                Page<T> results = jpaRepository.findAll(page);
                results.map(result -> {
                    // if there are any relationships to load, do it now
                    relationshipGetters.forEach(method -> {
                        try {
                            // eagerly load the relationship set
                            ((Set) method.invoke(result)).size();
                        } catch (Exception ex) {
                            log.error(ex.getMessage());
                        }
                    });
                    return result;
                });

                elasticsearchRepository.saveAll(results.getContent());
            }
        }
        log.info("Elasticsearch: Indexed all rows for {}", entityClass.getSimpleName());
    }
}

deepu105 commented 2 years ago

@dungreact please ask to reopen if its still an issue

jhipster / generator-jhipster