Had an epiphany to try writing a working Python script using Copilot. My prompt was basically requesting a Python script to ask the user to input a URL to search for either "robot.txt" or "robots.txt" and saving the file with the URL appended to the filename.
The Python code is as follows.
import requests
def check_robots_txt(url):
# Ensure the URL starts with http:// or https://
if not url.startswith(('http://', 'https://')):
url = 'http://' + url
# Check for robots.txt
robots_url = url.rstrip('/') + '/robots.txt'
response = requests.get(robots_url)
if response.status_code == 200:
print(f"Found robots.txt at {robots_url}")
print("Contents of robots.txt:")
print(response.text)
filename = f"{url.replace('http://', '').replace('https://', '').replace('/', '_')}_robots.txt"
with open(filename, 'w') as file:
file.write(response.text)
print(f"The contents have been saved to {filename}")
else:
print(f"No robots.txt found at {robots_url}")
# Check for robot.txt
robot_url = url.rstrip('/') + '/robot.txt'
response = requests.get(robot_url)
if response.status_code == 200:
print(f"Found robot.txt at {robot_url}")
print("Contents of robot.txt:")
print(response.text)
filename = f"{url.replace('http://', '').replace('https://', '').replace('/', '_')}_robot.txt"
with open(filename, 'w') as file:
file.write(response.text)
print(f"The contents have been saved to {filename}")
else:
print(f"No robot.txt found at {robot_url}")
# Input URL from the user
url = input("Enter the URL: ")
check_robots_txt(url)
Worked like a charm.
