#_________________________________________________ IMPORT LIBRARY _________________________________________________
import re
import tweepy
import pandas as pd
from numpy import append
from datetime import datetime, timedelta, timezone

#_________________________________________________ API TWITTER _________________________________________________
# API 1
consumer_key = "Ix35P00OPyr4V2q8bufIoEaAD" #Your API/Consumer key
consumer_secret = "isNQqFLN9D7NYYjFoBKK5QO3scIW9ppUcrhyj8rCLxlu005RK4" #Your API/Consumer Secret Key
access_token = "541678481-XK8MjijpRbTZCBvFxsZvhJr1VI5YKxfaOJbE5Gvn"    #Your Access token key
access_token_secret = "8MrZssjxJ7KLDiwbd2iB5HT470H4fytctOmfYRyrRS6EH" #Your Access token Secret key
client2 = tweepy.Client(consumer_key=consumer_key, consumer_secret=consumer_secret, access_token=access_token, access_token_secret=access_token_secret, wait_on_rate_limit=True)
# API 2
client = tweepy.Client(bearer_token='AAAAAAAAAAAAAAAAAAAAAN%2BqyQEAAAAAd%2B7%2Fctp9LZ%2FT3ifZu6lsCn4m5vQ%3DtKBf819ulXzt5rv2t1KZTMnm2YGon48ezvjxPEzAksVZptDlaB')

#_________________________________________________ DEFINISIKAN FUNGSI _________________________________________________
satuan = {"M": " juta",
        "K":" ribu",
        "B":" miliar"
        }        
def satuan_eng_to_id(str_text):
  for i in satuan:
    str_text = str_text.replace(i, satuan[i])
  return str_text

def clean_twitter_text(text):
  text = re.sub(r'@\w+', '', text)
  text = re.sub(r'#\w+', '', text)
  text = re.sub(r'RT[\s]+', '', text)
  text = re.sub(r'https?://\S+', '', text)

  text = re.sub(r'[^A-Za-z0-9 ]', '', text)
  text = re.sub(r'\s+', ' ', text).strip()

  return text



#_________________________________________________ DEFINISIKAN FUNGSI SCRAPPING TWITTER _________________________________________________
def scrap_twt(limit,nama,start_time,end_time):
  query = (nama + ' -is:retweet -is:reply -is:quote lang:id')
  if limit>100:
    max_results = 100
    limit_result = limit//100
  elif limit<10:
    max_results = 10
    limit_result = 1
  else:
    max_results = limit
    limit_result = 1

  result = []; tweets = []
  for response in tweepy.Paginator(
      client.search_recent_tweets
      , query = query
      , start_time=start_time
      , end_time=end_time
      , tweet_fields=['id', 'referenced_tweets','text','context_annotations','created_at','geo','author_id','lang','public_metrics','source']
      , user_fields =['id','name','username','url','public_metrics']
      #, place_fields = ['contained_within', 'country', 'country_code', 'full_name', 'geo', 'id', 'name', 'place_type']
      , expansions=["referenced_tweets.id","author_id","geo.place_id",'referenced_tweets.id.author_id']
      ,max_results = max_results
      ,limit=limit_result):

      usersdict = {x.id:append(x.username,x.name) for x in response.includes['users']}
      folusersdict = {x.id:x.public_metrics for x in response.includes['users']}
      
      for tweet in response.data:
          tweets.append(tweet)

          result.append({'id': tweet.id,
                        'author_id': tweet.author_id,
                        'username': usersdict[tweet.author_id][0],
                        'accname': usersdict[tweet.author_id][1],
                        'followers_count': folusersdict[tweet.author_id]['followers_count'],
                        'following_count': folusersdict[tweet.author_id]['following_count'],
                        'post_count': folusersdict[tweet.author_id]['tweet_count'],
                        'listed_count': folusersdict[tweet.author_id]['listed_count'],
                        'like_count': folusersdict[tweet.author_id]['like_count'],
                        'media_count': folusersdict[tweet.author_id]['media_count'],
                        'user_url': f"https://twitter.com/{usersdict[tweet.author_id][0]}",
                        'lang': tweet.lang,
                        'text': tweet.text,
                        'created_at': tweet.created_at,
                        'source':tweet.source,
                        'retweets': tweet.public_metrics['retweet_count'],
                        'replies': tweet.public_metrics['reply_count'],
                        'likes': tweet.public_metrics['like_count'],
                        'quote_count': tweet.public_metrics['quote_count'],
                        'bookmark_count' : tweet.public_metrics['bookmark_count'],
                        'impression_count': tweet.public_metrics['impression_count'],
                        'referenced_tweets': tweet.referenced_tweets,
                        #'context_annotations': tweet.context_annotations,
                        'geo': tweet.geo,
                        'link_url': f"https://twitter.com/{usersdict[tweet.author_id][0]}/status/{tweet.id}"
                    })
  df = pd.DataFrame(result)
  df['hastag'] = df.text.str.findall(r'#.*?(?=\s|$)')
  df['created_at'] = pd.to_datetime(df['created_at']).dt.tz_localize(None) + timedelta(hours=7)
  df['perjam'] = (df['created_at'].dt.strftime("%Y/%m/%d %H"))
  df['perhari'] = df['created_at'].dt.strftime("%Y/%m/%d")
  df['tahun'] = df['created_at'].dt.strftime("%Y")
  df['bulan'] = df['created_at'].dt.strftime("%m")
  df['hari'] = df['created_at'].dt.strftime("%d")
  df['SUMBER'] = "TWITTER"
  df['keyword'] = nama
  return df


#_________________________________________________ SENTIMENT ANALYSIS _________________________________________________
import os
from pickle import TRUE
import re
from numpy import append

#_________________________________________________ DEFINISIKAN FUNGSI _________________________________________________
satuan = {"M": " juta",
        "K":" ribu",
        "B":" miliar"
        }        
def satuan_eng_to_id(str_text):
  for i in satuan:
    str_text = str_text.replace(i, satuan[i])
  return str_text

def clean_twitter_text(text):
  text = re.sub(r'@\w+', '', text) #Menghapus mention
  text = re.sub(r'#\w+', '', text) #Menghapus hashtag
  text = re.sub(r'RT[\s]+', '', text) #Menghapus RT
  text = re.sub(r'https?://\S+', '', text) #Menghapus link
  text = re.sub(r'[^A-Za-z0-9 ]', '', text) #Menghapus karakter selain huruf dan angka
  text = re.sub(r'\s+', ' ', text).strip() #Menghapus spasi berlebih
  return text

def normalisasi(str_text):
  norm = {}
  with open("/home/mazhters/ujipython.mazhters.com/sumber/normalisasi.txt", "r") as file:
    for line in file:
      line = line.replace('\n', '').replace('\r\n', '')
      norm.update({line.split(':')[0]:line.split(':')[1]})
  for i in norm:
    str_text = str_text.replace(i, norm[i])
  return str_text


def translasi_text(text):  
    from googletrans import Translator
    translator = Translator()
    tr_en = translator.translate(text, src='id', dest='en').text
    return tr_en

def remove_stopwords(tokens):
    from nltk.corpus import stopwords
    stop_words = set(stopwords.words('english'))
    stop_words.add('.')
    return [word for word in tokens if word.lower() not in stop_words]


def stemen(words):
  from nltk.stem import PorterStemmer
  ps = PorterStemmer()
  filtered_sentence = ''
  for w in words:
      filtered_sentence = filtered_sentence + ' ' + ps.stem(w)
  return filtered_sentence.strip()


def sentimensc(words2):
  from nltk.sentiment.vader import SentimentIntensityAnalyzer
  sid = SentimentIntensityAnalyzer()
  sentiment_scores = sid.polarity_scores(words2)
  return sentiment_scores