from google.cloud import vectorsearch_v1beta# ConfigurationPROJECT_ID = "your-project-id"LOCATION = "us-central1"COLLECTION_ID = "langchain-test-collection"# Create the Vector Search service clientvector_search_service_client = vectorsearch_v1beta.VectorSearchServiceClient()# Create the collection with schema compatible with LangChain# IMPORTANT: To enable filtering, you must define filterable fields in data_schema.propertiesrequest = vectorsearch_v1beta.CreateCollectionRequest( parent=f"projects/{PROJECT_ID}/locations/{LOCATION}", collection_id=COLLECTION_ID, collection={ "display_name": "LangChain Test Collection", "description": "Collection for testing LangChain VectorSearchVectorStore with filtering", "data_schema": { "type": "object", "properties": { # Define fields you want to filter on "source": {"type": "string"}, "category": {"type": "string"}, "page": {"type": "number"}, # Add more fields as needed for your specific use case }, }, "vector_schema": { # Vector field must be named "embedding" to match LangChain's default "embedding": { "dense_vector": { "dimensions": 768 # For text-embedding-005 } }, }, },)print(f"Creating collection: {COLLECTION_ID}")operation = vector_search_service_client.create_collection(request=request)print(f"Operation started: {operation.operation.name}")print("Waiting for operation to complete...")result = operation.result()print(f"Collection created successfully!")print(f"Resource name: {result.name}")
from langchain_google_vertexai import VectorSearchVectorStore, VertexAIEmbeddings# Initialize embeddingsembeddings = VertexAIEmbeddings(model_name="text-embedding-005")# Create vector store from a Collection# Use the same PROJECT_ID, LOCATION, and COLLECTION_ID from collection creationvector_store = VectorSearchVectorStore.from_components( project_id=PROJECT_ID, region=LOCATION, collection_id=COLLECTION_ID, embedding=embeddings, api_version="v2",)
# Search using a pre-computed embeddingembedding = embeddings.embed_query("Feature Store on Gemini Enterprise Agent Platform")results = vector_store.similarity_search_by_vector_with_score(embedding, k=5)for doc, score in results: print(f"Score: {score}") print(f"Content: {doc.page_content}\n")
# Recommended: Search first, then delete by IDsresults = vector_store.similarity_search( "query", # Use a broad query k=1000, # Get more results filter={"source": {"$eq": "old_docs"}})ids_to_delete = [doc.metadata.get("id") for doc in results if "id" in doc.metadata]vector_store.delete(ids=ids_to_delete)
或者,如果您的环境支持直接删除元数据:
# Direct deletion by metadata (may have limitations)try: vector_store.delete(metadata={"source": {"$eq": "old_docs"}})except Exception as e: # Fall back to search-then-delete approach print(f"Direct deletion failed: {e}")
# TODO : Set values as per your requirements# Project and Storage ConstantsPROJECT_ID = "<my_project_id>"REGION = "<my_region>"BUCKET = "<my_gcs_bucket>"BUCKET_URI = f"gs://{BUCKET}"# The number of dimensions for the textembedding-gecko@003 is 768# If other embedder is used, the dimensions would probably need to change.DIMENSIONS = 768# Index ConstantsDISPLAY_NAME = "<my_matching_engine_index_id>"DEPLOYED_INDEX_ID = "<my_matching_engine_endpoint_id>"
# NOTE : This operation can take upto 20 minutesmy_index_endpoint = my_index_endpoint.deploy_index( index=my_index, deployed_index_id=DEPLOYED_INDEX_ID)my_index_endpoint.deployed_indexes
# TODO : replace 1234567890123456789 with your actual index IDmy_index = aiplatform.MatchingEngineIndex("1234567890123456789")# TODO : replace 1234567890123456789 with your actual endpoint IDmy_index_endpoint = aiplatform.MatchingEngineIndexEndpoint("1234567890123456789")
from langchain_google_vertexai import ( VectorSearchVectorStore, VectorSearchVectorStoreDatastore,)
# Input textstexts = [ "The cat sat on", "the mat.", "I like to", "eat pizza for", "dinner.", "The sun sets", "in the west.",]# Create a Vector Storevector_store = VectorSearchVectorStore.from_components( project_id=PROJECT_ID, region=REGION, gcs_bucket_name=BUCKET, index_id=my_index.name, endpoint_id=my_index_endpoint.name, embedding=embedding_model, stream_update=True,)# Add vectors and mapped text chunks to your vector storevector_store.add_texts(texts=texts)
# Input text with metadatarecord_data = [ { "description": "A versatile pair of dark-wash denim jeans." "Made from durable cotton with a classic straight-leg cut, these jeans" " transition easily from casual days to dressier occasions.", "price": 65.00, "color": "blue", "season": ["fall", "winter", "spring"], }, { "description": "A lightweight linen button-down shirt in a crisp white." " Perfect for keeping cool with breathable fabric and a relaxed fit.", "price": 34.99, "color": "white", "season": ["summer", "spring"], }, { "description": "A soft, chunky knit sweater in a vibrant forest green. " "The oversized fit and cozy wool blend make this ideal for staying warm " "when the temperature drops.", "price": 89.99, "color": "green", "season": ["fall", "winter"], }, { "description": "A classic crewneck t-shirt in a soft, heathered blue. " "Made from comfortable cotton jersey, this t-shirt is a wardrobe essential " "that works for every season.", "price": 19.99, "color": "blue", "season": ["fall", "winter", "summer", "spring"], }, { "description": "A flowing midi-skirt in a delicate floral print. " "Lightweight and airy, this skirt adds a touch of feminine style " "to warmer days.", "price": 45.00, "color": "white", "season": ["spring", "summer"], },]
# Parse and prepare input datatexts = []metadatas = []for record in record_data: record = record.copy() page_content = record.pop("description") texts.append(page_content) if isinstance(page_content, str): metadata = {**record} metadatas.append(metadata)
# Inspect metadatasmetadatas
# NOTE : This operation can take more than 20 minsvector_store = VectorSearchVectorStore.from_components( project_id=PROJECT_ID, region=REGION, gcs_bucket_name=BUCKET, index_id=my_index.name, endpoint_id=my_index_endpoint.name, embedding=embedding_model,)vector_store.add_texts(texts=texts, metadatas=metadatas, is_complete_overwrite=True)
from google.cloud.aiplatform.matching_engine.matching_engine_index_endpoint import ( Namespace, NumericNamespace,)
# Try running a simple similarity search# Below code should return 5 resultsvector_store.similarity_search("shirt", k=5)
# Try running a similarity search with text filterfilters = [Namespace(name="season", allow_tokens=["spring"])]# Below code should return 4 results nowvector_store.similarity_search("shirt", k=5, filter=filters)
# Try running a similarity search with combination of text and numeric filterfilters = [Namespace(name="season", allow_tokens=["spring"])]numeric_filters = [NumericNamespace(name="price", value_float=40.0, op="LESS")]# Below code should return 2 results nowvector_store.similarity_search( "shirt", k=5, filter=filters, numeric_filter=numeric_filters)
# Initialize the vectore_store as retrieverretriever = vector_store.as_retriever()
# perform simple similarity search on retrieverretriever.invoke("What are my options in breathable fabric?")
# Try running a similarity search with text filterfilters = [Namespace(name="season", allow_tokens=["spring"])]retriever.search_kwargs = {"filter": filters}# perform similarity search with filters on retrieverretriever.invoke("What are my options in breathable fabric?")
# Try running a similarity search with combination of text and numeric filterfilters = [Namespace(name="season", allow_tokens=["spring"])]numeric_filters = [NumericNamespace(name="price", value_float=40.0, op="LESS")]retriever.search_kwargs = {"filter": filters, "numeric_filter": numeric_filters}retriever.invoke("What are my options in breathable fabric?")
text_splitter = RecursiveCharacterTextSplitter( # Set a really small chunk size, just to show. chunk_size=1000, chunk_overlap=20, length_function=len, is_separator_regex=False,)doc_splits = text_splitter.split_documents(pages)
texts = [doc.page_content for doc in doc_splits]metadatas = [doc.metadata for doc in doc_splits]
# Define some sample datatexts = [ "The cat sat on", "the mat.", "I like to", "eat pizza for", "dinner.", "The sun sets", "in the west.",]# optional IDsids = ["i_" + str(i + 1) for i in range(len(texts))]# optional metadatametadatas = [{"my_metadata": i} for i in range(len(texts))]
from sklearn.feature_extraction.text import TfidfVectorizer# Fit the TFIDF Vectorizer (This is usually done on a very large corpus of data to make sure that word statistics generalize well on new data)vectorizer = TfidfVectorizer()vectorizer.fit(texts)
# Utility function to transform text into a TF-IDF Sparse Vectordef get_sparse_embedding(tfidf_vectorizer, text): tfidf_vector = tfidf_vectorizer.transform([text]) values = [] dims = [] for i, tfidf_value in enumerate(tfidf_vector.data): values.append(float(tfidf_value)) dims.append(int(tfidf_vector.indices[i])) return {"values": values, "dimensions": dims}
# semantic (dense) embeddingsembeddings = embedding_model.embed_documents(texts)# tfidf (sparse) embeddingssparse_embeddings = [get_sparse_embedding(vectorizer, x) for x in texts]
sparse_embeddings[0]
# Add the dense and sparse embeddings in Vector Searchvector_store.add_texts_with_embeddings( texts=texts, embeddings=embeddings, sparse_embeddings=sparse_embeddings, ids=ids, metadatas=metadatas,)
# Run hybrid searchquery = "the cat"embedding = embedding_model.embed_query(query)sparse_embedding = get_sparse_embedding(vectorizer, query)vector_store.similarity_search_by_vector_with_score( embedding=embedding, sparse_embedding=sparse_embedding, k=5, rrf_ranking_alpha=0.7, # 0.7 weight to dense and 0.3 weight to sparse)