Spaces:
Running
on
Zero
Running
on
Zero
add input check
Browse files- prompt_check.py +0 -7
prompt_check.py
CHANGED
|
@@ -1,7 +1,3 @@
|
|
| 1 |
-
from diffusers.utils.logging import get_logger
|
| 2 |
-
|
| 3 |
-
logger = get_logger(__name__)
|
| 4 |
-
|
| 5 |
def is_unsafe_prompt(model, tokenizer, user_prompt):
|
| 6 |
prompt = f"""You are a content safety filter. Your task is to analyze the following user input and determine if it contains unsafe content.
|
| 7 |
|
|
@@ -20,7 +16,6 @@ def is_unsafe_prompt(model, tokenizer, user_prompt):
|
|
| 20 |
{"role": "user", "content": prompt}
|
| 21 |
]
|
| 22 |
|
| 23 |
-
logger.warning(str(messages))
|
| 24 |
text = tokenizer.apply_chat_template(
|
| 25 |
messages,
|
| 26 |
tokenize=False,
|
|
@@ -38,6 +33,4 @@ def is_unsafe_prompt(model, tokenizer, user_prompt):
|
|
| 38 |
|
| 39 |
content = tokenizer.decode(output_ids, skip_special_tokens=True).strip("\n")
|
| 40 |
|
| 41 |
-
logger.warning(content)
|
| 42 |
-
|
| 43 |
return "yes" in content.lower()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
def is_unsafe_prompt(model, tokenizer, user_prompt):
|
| 2 |
prompt = f"""You are a content safety filter. Your task is to analyze the following user input and determine if it contains unsafe content.
|
| 3 |
|
|
|
|
| 16 |
{"role": "user", "content": prompt}
|
| 17 |
]
|
| 18 |
|
|
|
|
| 19 |
text = tokenizer.apply_chat_template(
|
| 20 |
messages,
|
| 21 |
tokenize=False,
|
|
|
|
| 33 |
|
| 34 |
content = tokenizer.decode(output_ids, skip_special_tokens=True).strip("\n")
|
| 35 |
|
|
|
|
|
|
|
| 36 |
return "yes" in content.lower()
|